diff options
Diffstat (limited to 'lib')
132 files changed, 1921 insertions, 554 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 982f45425a3..684955a1b24 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -231,6 +231,20 @@ module API render_api_error!("Failed to save note #{note.errors.messages}", 400) end end + + desc 'Get Merge Requests associated with a commit' do + success Entities::MergeRequestBasic + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to find Merge Requests' + use :pagination + end + get ':id/repository/commits/:sha/merge_requests', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + present paginate(commit.merge_requests), with: Entities::MergeRequestBasic + end end end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b0b7b50998f..70d43ac1d79 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -54,7 +54,7 @@ module API present key, with: Entities::DeployKeysProject end - desc 'Add new deploy key to currently authenticated user' do + desc 'Add new deploy key to a project' do success Entities::DeployKeysProject end params do @@ -66,33 +66,32 @@ module API params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys_projects + deploy_key_project = user_project.deploy_keys_projects .joins(:deploy_key) .find_by(keys: { key: params[:key] }) - if key - present key, with: Entities::DeployKeysProject + if deploy_key_project + present deploy_key_project, with: Entities::DeployKeysProject break end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key - added_key = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) + deploy_key_project = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) - present added_key, with: Entities::DeployKeysProject + present deploy_key_project, with: Entities::DeployKeysProject break end # Create a new deploy key - key_attributes = { can_push: !!params[:can_push], - deploy_key_attributes: declared_params.except(:can_push) } - key = add_deploy_keys_project(user_project, key_attributes) + deploy_key_attributes = declared_params.except(:can_push).merge(user: current_user) + deploy_key_project = add_deploy_keys_project(user_project, deploy_key_attributes: deploy_key_attributes, can_push: !!params[:can_push]) - if key.valid? - present key, with: Entities::DeployKeysProject + if deploy_key_project.valid? + present deploy_key_project, with: Entities::DeployKeysProject else - render_validation_error!(key) + render_validation_error!(deploy_key_project) end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 16147ee90c9..e5ecd37e473 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -206,6 +206,7 @@ module API expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved expose :printing_merge_request_link_enabled + expose :merge_method expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics @@ -405,6 +406,7 @@ module API class IssueBasic < ProjectEntity expose :closed_at + expose :closed_by, using: Entities::UserBasic expose :labels do |issue, options| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort @@ -951,6 +953,7 @@ module API expose :tag_list expose :run_untagged expose :locked + expose :maximum_timeout expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at @@ -1119,7 +1122,7 @@ module API end class RunnerInfo < Grape::Entity - expose :timeout + expose :metadata_timeout, as: :timeout end class Step < Grape::Entity diff --git a/lib/api/features.rb b/lib/api/features.rb index 9385c6ca174..11d848584d9 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -65,6 +65,13 @@ module API present feature, with: Entities::Feature, current_user: current_user end + + desc 'Remove the gate value for the given feature' + delete ':name' do + Feature.get(params[:name]).remove + + status 204 + end end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e4fca77ab5d..61c138a7dec 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -83,12 +83,13 @@ module API end def available_labels_for(label_parent) - search_params = - if label_parent.is_a?(Project) - { project_id: label_parent.id } - else - { group_id: label_parent.id, only_group_labels: true } - end + search_params = { include_ancestor_groups: true } + + if label_parent.is_a?(Project) + search_params[:project_id] = label_parent.id + else + search_params.merge!(group_id: label_parent.id, only_group_labels: true) + end LabelsFinder.new(current_user, search_params).execute end @@ -410,7 +411,7 @@ module API ) end - def present_file!(path, filename, content_type = 'application/octet-stream') + def present_disk_file!(path, filename, content_type = 'application/octet-stream') filename ||= File.basename(path) header['Content-Disposition'] = "attachment; filename=#{filename}" header['Content-Transfer-Encoding'] = 'binary' @@ -426,13 +427,17 @@ module API end end - def present_artifacts!(artifacts_file) - return not_found! unless artifacts_file.exists? + def present_carrierwave_file!(file, supports_direct_download: true) + return not_found! unless file.exists? - if artifacts_file.file_storage? - present_file!(artifacts_file.path, artifacts_file.filename) + if file.file_storage? + present_disk_file!(file.path, file.filename) + elsif supports_direct_download && file.class.direct_download_enabled? + redirect(file.url) else - redirect_to(artifacts_file.url) + header(*Gitlab::Workhorse.send_url(file.url)) + status :ok + body end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 14648588dfd..abe3d353984 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -29,18 +29,6 @@ module API {} end - def fix_git_env_repository_paths(env, repository_path) - if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence - env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) - end - - if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) } - end - - env - end - def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS diff --git a/lib/api/internal.rb b/lib/api/internal.rb index b3660e4a1d0..fcbc248fc3b 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -21,8 +21,7 @@ module API # Stores some Git-specific env thread-safely env = parse_env - env = fix_git_env_repository_paths(env, repository_path) if project - Gitlab::Git::Env.set(env) + Gitlab::Git::HookEnv.set(gl_repository, env) if project actor = if params[:key_id] diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 47e5eeab31d..b1adef49d46 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -28,7 +28,7 @@ module API builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) - present_artifacts!(latest_build.artifacts_file) + present_carrierwave_file!(latest_build.artifacts_file) end desc 'Download the artifacts archive from a job' do @@ -43,7 +43,7 @@ module API build = find_build!(params[:job_id]) - present_artifacts!(build.artifacts_file) + present_carrierwave_file!(build.artifacts_file) end desc 'Download a specific file from artifacts archive' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 9c205514b3a..60911c8d733 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -72,7 +72,7 @@ module API present build, with: Entities::Job end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. desc 'Get a trace of a specific job of a project' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index b0a7fd6f4ab..5ef4e9d530c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API render_api_error!('404 Not found or has expired', 404) unless path - present_file!(path, File.basename(path), 'application/gzip') + present_disk_file!(path, File.basename(path), 'application/gzip') end desc 'Start export' do @@ -33,11 +33,28 @@ module API end params do optional :description, type: String, desc: 'Override the project description' + optional :upload, type: Hash do + optional :url, type: String, desc: 'The URL to upload the project' + optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project' + end end post ':id/export' do project_export_params = declared_params(include_missing: false) + after_export_params = project_export_params.delete(:upload) || {} - user_project.add_export_job(current_user: current_user, params: project_export_params) + export_strategy = if after_export_params[:url].present? + params = after_export_params.slice(:url, :http_method).symbolize_keys + + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + end + + if export_strategy&.invalid? + render_validation_error!(export_strategy) + else + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + end accepted! end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index b552b0e0c5d..3d5b3c5a535 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -28,6 +28,7 @@ module API optional :tag_list, type: Array[String], desc: 'The list of tags for a project' optional :avatar, type: File, desc: 'Avatar image for project' optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' + optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' end params :optional_params do @@ -228,11 +229,7 @@ module API namespace_id = fork_params[:namespace] if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end + fork_params[:namespace] = find_namespace(namespace_id) unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') @@ -278,6 +275,7 @@ module API :issues_enabled, :lfs_enabled, :merge_requests_enabled, + :merge_method, :name, :only_allow_merge_if_all_discussions_are_resolved, :only_allow_merge_if_pipeline_succeeds, diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index c15c487deb4..aa7cab4a741 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -52,11 +52,7 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - # Replace with `declared(params)` after updating to grape v1.0.2 - # See https://github.com/ruby-grape/grape/pull/1710 - # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 - declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - + declared_params = declared_params(include_missing: false) api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) protected_branch = api_service.create @@ -74,7 +70,10 @@ module API delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) - destroy_conditionally!(protected_branch) + destroy_conditionally!(protected_branch) do + destroy_service = ::ProtectedBranches::DestroyService.new(user_project, current_user) + destroy_service.execute(protected_branch) + end end end end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 7e6c33ec33d..834253d8e94 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -14,9 +14,10 @@ module API optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs' optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) + optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end post '/' do - attributes = attributes_for_keys([:description, :locked, :run_untagged, :tag_list]) + attributes = attributes_for_keys([:description, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) runner = @@ -207,6 +208,7 @@ module API optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of the file) end post '/:id/artifacts' do not_allowed! unless Gitlab.config.artifacts.enabled @@ -226,7 +228,7 @@ module API Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) - job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata + job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, file_sha256: params['metadata.sha256'], expire_in: expire_in) if metadata job.artifacts_expire_in = expire_in if job.save @@ -244,11 +246,12 @@ module API params do requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) + optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end get '/:id/artifacts' do job = authenticate_job! - present_artifacts!(job.artifacts_file) + present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 996457c5dfe..5f2a9567605 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -57,6 +57,7 @@ module API optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' optional :access_level, type: String, values: Ci::Runner.access_levels.keys, desc: 'The access_level of the runner' + optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index ac76fece931..683b9c993cb 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -85,7 +85,7 @@ module API build = get_build!(params[:build_id]) - present_artifacts!(build.artifacts_file) + present_carrierwave_file!(build.artifacts_file) end desc 'Download the artifacts file from build' do @@ -102,10 +102,10 @@ module API builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) - present_artifacts!(latest_build.artifacts_file) + present_carrierwave_file!(latest_build.artifacts_file) end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. desc 'Get a trace of a specific build of a project' diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 7d8b1f369fe..a2df969d819 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -268,11 +268,7 @@ module API namespace_id = fork_params[:namespace] if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end + fork_params[:namespace] = find_namespace(namespace_id) unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 4383124d150..6a5a223a614 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -5,9 +5,5 @@ module Backup def initialize super('artifacts', JobArtifactUploader.root) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 635967f4bd4..f869916e199 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -5,9 +5,5 @@ module Backup def initialize super('builds', Settings.gitlab_ci.builds_path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 287d591e88d..88cb7e7b5a4 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,7 +1,10 @@ require 'open3' +require_relative 'helper' module Backup class Files + include Backup::Helper + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(name, app_files_dir) @@ -35,15 +38,22 @@ module Backup def restore backup_existing_files_dir - create_files_dir - run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir - timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}") if File.exist?(app_files_dir) - FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + FileUtils.mkdir_p(timestamped_files_path, mode: 0700) + files = Dir.glob(File.join(app_files_dir, "*"), File::FNM_DOTMATCH) - [File.join(app_files_dir, "."), File.join(app_files_dir, "..")] + begin + FileUtils.mv(files, timestamped_files_path) + rescue Errno::EACCES + access_denied_error(app_files_dir) + end end end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb new file mode 100644 index 00000000000..a1ee0faefe9 --- /dev/null +++ b/lib/backup/helper.rb @@ -0,0 +1,17 @@ +module Backup + module Helper + def access_denied_error(path) + message = <<~EOS + + ### NOTICE ### + As part of restore, the task tried to move existing content from #{path}. + However, it seems that directory contains files/folders that are not owned + by the user #{Gitlab.config.gitlab.user}. To proceed, please move the files + or folders inside #{path} to a secure location so that #{path} is empty and + run restore task again. + + EOS + raise message + end + end +end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 4153467fbee..4e234e50a7a 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -5,9 +5,5 @@ module Backup def initialize super('lfs', Settings.lfs.storage_path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 215ded93bfe..5830b209d6e 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -5,9 +5,5 @@ module Backup def initialize super('pages', Gitlab.config.pages.path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 67fe0231087..91698669402 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -5,9 +5,5 @@ module Backup def initialize super('registry', Settings.registry.path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6715159a1aa..89e3f1d9076 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,8 +1,11 @@ require 'yaml' +require_relative 'helper' module Backup class Repository + include Backup::Helper # rubocop:disable Metrics/AbcSize + def dump prepare @@ -63,18 +66,27 @@ module Backup end end - def restore + def prepare_directories Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage['path'] + path = repository_storage.legacy_disk_path next unless File.exist?(path) - # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(path, bk_repos_path) - # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 02770) + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + end end + end + def restore + prepare_directories Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) @@ -200,7 +212,7 @@ module Backup end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end def progress diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 35118375499..d46e2cd869d 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -5,9 +5,5 @@ module Backup def initialize super('uploads', Rails.root.join('public/uploads')) end - - def create_files_dir - Dir.mkdir(app_files_dir) - end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index c9e3f8ce42b..c3a03f13306 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -171,7 +171,7 @@ module Banzai end if object - title = object_link_title(object) + title = object_link_title(object, matches) klass = reference_class(object_sym) data = data_attributes_for(link_content || match, parent, object, @@ -216,7 +216,7 @@ module Banzai extras end - def object_link_title(object) + def object_link_title(object, matches) object.title end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 75b64ae9af2..4a143baeef6 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -21,12 +21,13 @@ module Banzai # # See http://en.wikipedia.org/wiki/URI_scheme # - # The negative lookbehind ensures that users can paste a URL followed by a - # period or comma for punctuation without those characters being included - # in the generated link. + # The negative lookbehind ensures that users can paste a URL followed by + # punctuation without those characters being included in the generated + # link. It matches the behaviour of Rinku 2.0.1: + # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # - # Rubular: http://rubular.com/r/JzPhi6DCZp - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} + # Rubular: http://rubular.com/r/nrL3r9yUiq + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -104,8 +105,12 @@ module Banzai end end - options = link_options.merge(href: match) - content_tag(:a, match.html_safe, options) + dropped + # match has come from node.to_html above, so we know it's encoded + # correctly. + html_safe_match = match.html_safe + options = link_options.merge(href: html_safe_match) + + content_tag(:a, html_safe_match, options) + dropped end def autolink_filter(text) diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 21bcb1c5ca8..99fa2d9d8fb 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -34,7 +34,7 @@ module Banzai range.to_param.merge(only_path: context[:only_path])) end - def object_link_title(range) + def object_link_title(range, matches) nil end end diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb new file mode 100644 index 00000000000..ef16df1f3ae --- /dev/null +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -0,0 +1,152 @@ +module Banzai + module Filter + # HTML filter that replaces users' names and emails in commit trailers + # with links to their GitLab accounts or mailto links to their mentioned + # emails. + # + # Commit trailers are special labels in the form of `*-by:` and fall on a + # single line, ex: + # + # Reported-By: John S. Doe <john.doe@foo.bar> + # + # More info about this can be found here: + # * https://git.wiki.kernel.org/index.php/CommitMessageConventions + class CommitTrailersFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + include ApplicationHelper + include AvatarsHelper + + TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze + AUTHOR_REGEXP = /(?<author_name>.+)/.freeze + # Devise.email_regexp wouldn't work here since its designed to match + # against strings that only contains email addresses; the \A and \z + # around the expression will only match if the string being matched + # contains just the email nothing else. + MAIL_REGEXP = /<(?<author_email>[^@\s]+@[^@\s]+)>/.freeze + FILTER_REGEXP = /(?<trailer>^\s*#{TRAILER_REGEXP}\s*#{AUTHOR_REGEXP}\s+#{MAIL_REGEXP}$)/mi.freeze + + def call + doc.xpath('descendant-or-self::text()').each do |node| + content = node.to_html + + next unless content.match(FILTER_REGEXP) + + html = trailer_filter(content) + + next if html == content + + node.replace(html) + end + + doc + end + + private + + # Replace trailer lines with links to GitLab users or mailto links to + # non GitLab users. + # + # text - String text to replace trailers in. + # + # Returns a String with all trailer lines replaced with links to GitLab + # users and mailto links to non GitLab users. All links have `data-trailer` + # and `data-user` attributes attached. + def trailer_filter(text) + text.gsub(FILTER_REGEXP) do |author_match| + label = $~[:label] + "#{label} #{parse_user($~[:author_name], $~[:author_email], label)}" + end + end + + # Find a GitLab user using the supplied email and generate + # a valid link to them, otherwise, generate a mailto link. + # + # name - String name used in the commit message for the user + # email - String email used in the commit message for the user + # trailer - String trailer used in the commit message + # + # Returns a String with a link to the user. + def parse_user(name, email, trailer) + link_to_user User.find_by_any_email(email), + name: name, + email: email, + trailer: trailer + end + + def urls + Gitlab::Routing.url_helpers + end + + def link_to_user(user, name:, email:, trailer:) + wrapper = link_wrapper(data: { + trailer: trailer, + user: user.try(:id) + }) + + avatar = user_avatar_without_link( + user: user, + user_email: email, + css_class: 'avatar-inline', + has_tooltip: false + ) + + link_href = user.nil? ? "mailto:#{email}" : urls.user_url(user) + + avatar_link = link_tag( + link_href, + content: avatar, + title: email + ) + + name_link = link_tag( + link_href, + content: name, + title: email + ) + + email_link = link_tag( + "mailto:#{email}", + content: email, + title: email + ) + + wrapper << "#{avatar_link}#{name_link} <#{email_link}>" + end + + def link_wrapper(data: {}) + data_attributes = data_attributes_from_hash(data) + + doc.document.create_element( + 'span', + data_attributes + ) + end + + def link_tag(url, title: "", content: "", data: {}) + data_attributes = data_attributes_from_hash(data) + + attributes = data_attributes.merge( + href: url, + title: title + ) + + link = doc.document.create_element('a', attributes) + + if content.html_safe? + link << content + else + link.content = content # make sure we escape content using nokogiri's #content= + end + + link + end + + def data_attributes_from_hash(data = {}) + data.reject! {|_, value| value.nil?} + data.map do |key, value| + [%(data-#{key.to_s.dasherize}), value] + end.to_h + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index b82c6ca6393..e1261e7bbbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -11,7 +11,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| content = node.to_html next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index c2b42673376..f2e9a5a1116 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -57,7 +57,7 @@ module Banzai ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it # needs special-case handling diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index beb21b19ab3..73e82a4d7e3 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -4,7 +4,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) content = node.to_html diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 77299abe324..8f541dcfdb2 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -17,7 +17,7 @@ module Banzai issuables.each do |node, issuable| next if !can_read_cross_project? && issuable.project != project - if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" end end @@ -27,6 +27,10 @@ module Banzai private + def issuable_reference?(text, issuable) + text == issuable.reference_link_text(project || group) + end + def can_read_cross_project? Ability.allowed?(current_user, :read_cross_project) end @@ -38,6 +42,10 @@ module Banzai def project context[:project] end + + def group + context[:group] + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index d5360ad8f68..faa5b344e6f 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -41,7 +41,7 @@ module Banzai end def find_labels(project) - LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) + LabelsFinder.new(nil, project_id: project.id, include_ancestor_groups: true).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -77,7 +77,7 @@ module Banzai CGI.unescapeHTML(text.to_s) end - def object_link_title(object) + def object_link_title(object, matches) # use title of wrapped element instead nil end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index b3cfa97d0e0..5cbdb01c130 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -17,10 +17,19 @@ module Banzai only_path: context[:only_path]) end + def object_link_title(object, matches) + object_link_commit_title(object, matches) || super + end + def object_link_text_extras(object, matches) extras = super + if commit_ref = object_link_commit_ref(object, matches) + return extras.unshift(commit_ref) + end + path = matches[:path] if matches.names.include?("path") + case path when '/diffs' extras.unshift "diffs" @@ -38,6 +47,36 @@ module Banzai .where(iid: ids.to_a) .includes(target_project: :namespace) end + + private + + def object_link_commit_title(object, matches) + object_link_commit(object, matches)&.title + end + + def object_link_commit_ref(object, matches) + object_link_commit(object, matches)&.short_id + end + + def object_link_commit(object, matches) + return unless matches.names.include?('query') && query = matches[:query] + + # Removes leading "?". CGI.parse expects "arg1&arg2&arg3" + params = CGI.parse(query.sub(/^\?/, '')) + + return unless commit_sha = params['commit_id']&.first + + if commit = find_commit_by_sha(object, commit_sha) + Commit.from_hash(commit.to_hash, object.project) + end + end + + def find_commit_by_sha(object, commit_sha) + @all_commits ||= {} + @all_commits[object.id] ||= object.all_commits + + @all_commits[object.id].find { |commit| commit.sha == commit_sha } + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 8ec696ce5fc..1a1d7dbeb3d 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -84,7 +84,7 @@ module Banzai end end - def object_link_title(object) + def object_link_title(object, matches) nil end end diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb new file mode 100644 index 00000000000..607c2731ed3 --- /dev/null +++ b/lib/banzai/pipeline/commit_description_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class CommitDescriptionPipeline < SingleLinePipeline + def self.filters + @filters ||= super.concat FilterArray[ + Filter::CommitTrailersFilter, + ] + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index f5ccf952cf9..6af763faf10 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -69,7 +69,11 @@ module Gitlab authenticators.compact! - user if authenticators.find { |auth| auth.login(login, password) } + # return found user that was authenticated first for given login credentials + authenticators.find do |auth| + authenticated_user = auth.login(login, password) + break authenticated_user if authenticated_user + end end end diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index 260a77058a4..1234ace0334 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -8,7 +8,7 @@ module Gitlab def login(login, password) return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user&.valid_password?(password) + return user if user&.valid_password?(password) end end end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 77c0ddc2d48..34286900e72 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -52,6 +52,8 @@ module Gitlab block_user(user, 'does not exist anymore') false end + rescue LDAPConnectionError + false end def adapter diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index caf2d18c668..82ff1e77e5c 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -2,6 +2,9 @@ module Gitlab module Auth module LDAP class Adapter + SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + attr_reader :provider, :ldap def self.open(provider, &block) @@ -16,7 +19,7 @@ module Gitlab def initialize(provider, ldap = nil) @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) + @ldap = ldap || renew_connection_adapter end def config @@ -47,8 +50,10 @@ module Gitlab end def ldap_search(*args) + retries ||= 0 + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do + Timeout.timeout(timeout_time(retries)) do results = ldap.search(*args) if results.nil? @@ -63,16 +68,26 @@ module Gitlab results end end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] + rescue Net::LDAP::Error, Timeout::Error => error + retries += 1 + error_message = connection_error_message(error) + + Rails.logger.warn(error_message) + + if retries < MAX_SEARCH_RETRIES + renew_connection_adapter + retry + else + raise LDAPConnectionError, error_message + end end private + def timeout_time(retry_number) + SEARCH_RETRY_FACTOR[retry_number] * config.timeout + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), @@ -104,6 +119,18 @@ module Gitlab filter end end + + def connection_error_message(exception) + if exception.is_a?(Timeout::Error) + "LDAP search timed out after #{config.timeout} seconds" + else + "LDAP search raised exception #{exception.class}: #{exception.message}" + end + end + + def renew_connection_adapter + @ldap = Net::LDAP.new(config.adapter_options) + end end end end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index e70c3ab6b46..7c134fb6438 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -12,30 +12,26 @@ module Gitlab return unless Gitlab::Auth::LDAP::Config.enabled? return unless login.present? && password.present? - auth = nil - # loop through providers until valid bind + # return found user that was authenticated by first provider for given login credentials providers.find do |provider| auth = new(provider) - auth.login(login, password) # true will exit the loop + break auth.user if auth.login(login, password) # true will exit the loop end - - # If (login, password) was invalid for all providers, the value of auth is now the last - # Gitlab::Auth::LDAP::Authentication instance we tried. - auth.user end def self.providers Gitlab::Auth::LDAP::Config.providers end - attr_accessor :ldap_user - def login(login, password) - @ldap_user = adapter.bind_as( + result = adapter.bind_as( filter: user_filter(login), size: 1, password: password ) + return unless result + + @user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(result.dn, provider) end def adapter @@ -56,12 +52,6 @@ module Gitlab filter end - - def user - return unless ldap_user - - Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) - end end end end diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb new file mode 100644 index 00000000000..ef0a695742b --- /dev/null +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -0,0 +1,7 @@ +module Gitlab + module Auth + module LDAP + LDAPConnectionError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb index ed03b9f8b40..d4e7f35c857 100644 --- a/lib/gitlab/auth/o_auth/authentication.rb +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -12,6 +12,7 @@ module Gitlab @user = user end + # Implementation must return user object if login successful def login(login, password) raise NotImplementedError end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index b6a96081278..d0c6b0386ba 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -124,6 +124,9 @@ module Gitlab Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + + rescue Gitlab::Auth::LDAP::LDAPConnectionError + nil end def ldap_config diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 8fe4f1a2289..242e3143e71 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -12,6 +12,7 @@ module Gitlab class Build < ActiveRecord::Base self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled def ensure_stage!(attempts: 2) find_stage || create_stage! diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 884a3de8f62..1a25138e7d6 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - project.repository.create_hooks + Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index bffbcb86137..f3999e690fa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) + gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3263790a876..3a197078d08 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -9,20 +9,16 @@ module Gitlab super(project, user, protocol) end - def message(rejected: false) + def message <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: - #{remote_url_message(rejected)} + git remote set-url origin #{url_to_repo} MESSAGE end - def permanent_redirect? - RedirectRoute.permanent.exists?(path: redirected_path) - end - private attr_reader :redirected_path @@ -30,18 +26,6 @@ module Gitlab def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end - - def remote_url_message(rejected) - if rejected - "git remote set-url origin #{url_to_repo} and try again." - else - "git remote set-url origin #{url_to_repo}" - end - end - - def url - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end end end end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index b20d374288f..782f6c4c0af 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index eadc0948d2f..4aa5dc89f47 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index c317291f29d..f09ba42c074 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb new file mode 100644 index 00000000000..9d2a362b7d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Policy + class Variables < Policy::Specification + def initialize(expressions) + @expressions = Array(expressions) + end + + def satisfied_by?(pipeline, seed) + variables = seed.to_resource.scoped_variables_hash + + statements = @expressions.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement + .new(statement, variables) + end + + statements.any?(&:truthful?) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 411f67f8ce7..0b1ebe4e048 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -14,7 +14,7 @@ module Gitlab self.new(:script).tap do |step| step.script = job.options[:before_script].to_a + job.options[:script].to_a step.script = job.commands.split("\n") if step.script.empty? - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ON_SUCCESS end end @@ -25,7 +25,7 @@ module Gitlab self.new(:after_script).tap do |step| step.script = after_script - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ALWAYS step.allow_failure = true end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index f7ff7ea212e..66ac4a40616 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,7 +4,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - def initialize(config) + # EE would override this and utilize opts argument + def initialize(config, opts = {}) @config = Loader.new(config).load! @global = Entry::Global.new(@config) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0027e9ec8c5..09e8e52b60f 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,15 +25,31 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes + attributes :refs, :kubernetes, :variables validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes] + validates :config, allowed_keys: %i[refs kubernetes variables] + validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] + validates :variables, array_of_strings: true + end + + def variables_expressions_syntax + return unless variables.is_a?(Array) + + statements = variables.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement) + end + + statements.each do |statement| + unless statement.valid? + errors.add(:variables, "Invalid expression syntax") + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..f4c8d5342c1 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,11 +9,16 @@ module Gitlab ::Ci::Pipeline.transaction do pipeline.save! - @command.seeds_block&.call(pipeline) - - ::Ci::CreatePipelineStagesService - .new(project, current_user) - .execute(pipeline) + ## + # Create environments before the pipeline starts. + # + pipeline.builds.each do |build| + if build.has_environment? + project.environments.find_or_create_by( + name: build.expanded_environment_name + ) + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb new file mode 100644 index 00000000000..d299a5677de --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -0,0 +1,45 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Populate < Chain::Base + include Chain::Helpers + + PopulateError = Class.new(StandardError) + + def perform! + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Populate pipeline with all stages and builds from pipeline seeds. + # + pipeline.stage_seeds.each do |stage| + pipeline.stages << stage.to_resource + + stage.seeds.each do |build| + pipeline.builds << build.to_resource + end + end + + if pipeline.stages.none? + return error('No stages / jobs for this pipeline.') + end + + if pipeline.invalid? + return error('Failed to build the pipeline!') + end + + raise Populate::PopulateError if pipeline.persisted? + end + + def break? + pipeline.errors.any? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index 075504bcce5..a3bd2a5a23a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -16,11 +16,7 @@ module Gitlab @pipeline.drop!(:config_error) end - return error(@pipeline.yaml_errors) - end - - unless @pipeline.has_stage_seeds? - return error('No stages / jobs for this pipeline.') + error(@pipeline.yaml_errors) end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 48bde213d44..346c92dc51e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -4,7 +4,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) @value = value diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index b781c15fd67..37643c8ef53 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -11,7 +11,7 @@ module Gitlab end def evaluate(variables = {}) - HashWithIndifferentAccess.new(variables).fetch(@name, nil) + variables.with_indifferent_access.fetch(@name, nil) end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 4f0e101b730..09a7c98464b 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -14,12 +14,9 @@ module Gitlab %w[variable] ].freeze - def initialize(statement, pipeline) + def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) - - @variables = pipeline.variables.map do |variable| - [variable.key, variable.value] - end + @variables = variables.with_indifferent_access end def parse_tree @@ -35,6 +32,16 @@ module Gitlab def evaluate parse_tree.evaluate(@variables.to_h) end + + def truthful? + evaluate.present? + end + + def valid? + parse_tree.is_a?(Lexeme::Base) + rescue StatementError + false + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb new file mode 100644 index 00000000000..db9706924bb --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Base + def attributes + raise NotImplementedError + end + + def included? + raise NotImplementedError + end + + def to_resource + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb new file mode 100644 index 00000000000..6980b0b7aff --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -0,0 +1,48 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Build < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :dig, to: :@attributes + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @only = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:only)) + @except = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:except)) + end + + def included? + strong_memoize(:inclusion) do + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + end + end + + def attributes + @attributes.merge( + pipeline: @pipeline, + project: @pipeline.project, + user: @pipeline.user, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + ) + end + + def to_resource + strong_memoize(:resource) do + ::Ci::Build.new(attributes) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb new file mode 100644 index 00000000000..c101f30d6e8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -0,0 +1,47 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Stage < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :size, to: :seeds + delegate :dig, to: :seeds + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @builds = attributes.fetch(:builds).map do |attributes| + Seed::Build.new(@pipeline, attributes) + end + end + + def attributes + { name: @attributes.fetch(:name), + pipeline: @pipeline, + project: @pipeline.project } + end + + def seeds + strong_memoize(:seeds) do + @builds.select(&:included?) + end + end + + def included? + seeds.any? + end + + def to_resource + strong_memoize(:stage) do + ::Ci::Stage.new(attributes).tap do |stage| + seeds.each { |seed| stage.builds << seed.to_resource } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb deleted file mode 100644 index f33c87f554d..00000000000 --- a/lib/gitlab/ci/stage/seed.rb +++ /dev/null @@ -1,62 +0,0 @@ -module Gitlab - module Ci - module Stage - class Seed - include ::Gitlab::Utils::StrongMemoize - - attr_reader :pipeline - - delegate :project, to: :pipeline - delegate :size, to: :@jobs - - def initialize(pipeline, stage, jobs) - @pipeline = pipeline - @stage = { name: stage } - @jobs = jobs.to_a.dup - end - - def user=(current_user) - @jobs.map! do |attributes| - attributes.merge(user: current_user) - end - end - - def stage - @stage.merge(project: project) - end - - def builds - trigger = pipeline.trigger_requests.first - - @jobs.map do |attributes| - attributes.merge(project: project, - ref: pipeline.ref, - tag: pipeline.tag, - trigger_request: trigger, - protected: protected_ref?) - end - end - - def create! - pipeline.stages.create!(stage).tap do |stage| - builds_attributes = builds.map do |attributes| - attributes.merge(stage_id: stage.id) - end - - pipeline.builds.create!(builds_attributes).each do |build| - yield build if block_given? - end - end - end - - private - - def protected_ref? - strong_memoize(:protected_ref) do - project.protected_for?(pipeline.ref) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb new file mode 100644 index 00000000000..ac4308f4e2c --- /dev/null +++ b/lib/gitlab/ci/trace/http_io.rb @@ -0,0 +1,187 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil) + out = "" + + until eof? || (length && out.length >= length) + data = get_chunk + break if data.empty? + + out << data + @tell += data.bytesize + end + + out = out[0, length] if length && out.length > length + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.length)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index d52194f688b..b3fe3ef1c4d 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true delegate :valid?, to: :stream, as: :present?, allow_nil: true diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 0deca55fe8f..ad30b3f427c 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -30,7 +30,13 @@ module Gitlab end def to_runner_variables - self.map(&:to_hash) + self.map(&:to_runner_variable) + end + + def to_hash + self.to_runner_variables + .map { |env| [env.fetch(:key), env.fetch(:value)] } + .to_h.with_indifferent_access end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 939912981e6..23ed71db8b0 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,7 +17,7 @@ module Gitlab end def ==(other) - to_hash == self.class.fabricate(other).to_hash + to_runner_variable == self.class.fabricate(other).to_runner_variable end ## @@ -25,7 +25,7 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - def to_hash + def to_runner_variable @variable.reject do |hash_key, hash_value| hash_key == :file && hash_value == false end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a7285ac8f9d..e829f2a95f8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :cache, :stages, :jobs - def initialize(config) - @ci_config = Gitlab::Ci::Config.new(config) + def initialize(config, opts = {}) + @ci_config = Gitlab::Ci::Config.new(config, opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -27,7 +27,7 @@ module Gitlab end def build_attributes(name) - job = @jobs[name.to_sym] || {} + job = @jobs.fetch(name.to_sym, {}) { stage_idx: @stages.index(job[:stage]), stage: job[:stage], @@ -53,37 +53,31 @@ module Gitlab }.compact } end - def pipeline_stage_builds(stage, pipeline) - selected_jobs = @jobs.select do |_, job| - next unless job[:stage] == stage - - only_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:only, {})) - except_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:except, {})) - - only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(pipeline) } - end - - selected_jobs.map { |_, job| build_attributes(job[:name]) } + def stage_builds_attributes(stage) + @jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } end - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) + def stages_attributes + @stages.uniq.map do |stage| + seeds = stage_builds_attributes(stage).map do |attributes| + job = @jobs.fetch(attributes[:name].to_sym) - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end + attributes + .merge(only: job.fetch(:only, {})) + .merge(except: job.fetch(:except, {})) + end - seeds.compact + { name: stage, index: @stages.index(stage), builds: seeds } + end end - def self.validation_message(content) + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? begin - Gitlab::Ci::YamlProcessor.new(content) + Gitlab::Ci::YamlProcessor.new(content, opts) nil rescue ValidationError => e e.message diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 3ccfd9a739d..65a65b67975 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -40,7 +40,10 @@ module Gitlab # when there are no conflict files. files.each(&:lines) files.any? - rescue Gitlab::Git::CommandError, Gitlab::Git::Conflict::Parser::UnresolvableError, Gitlab::Git::Conflict::Resolver::ConflictSideMissing + rescue Gitlab::Git::CommandError, + Gitlab::Git::Conflict::Parser::UnresolvableError, + Gitlab::Git::Conflict::Resolver::ConflictSideMissing, + Gitlab::Git::Conflict::File::UnsupportedEncoding false end cache_method :can_be_resolved_in_ui? diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 44ca434056f..1634fe4e9cb 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -900,11 +900,42 @@ into similar problems in the future (e.g. when new tables are created). end end - # Rails' index_exists? doesn't work when you only give it a table and index - # name. As such we have to use some extra code to check if an index exists for - # a given name. + # Fetches indexes on a column by name for postgres. + # + # This will include indexes using an expression on the column, for example: + # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` + # + # For mysql, it falls back to the default ActiveRecord implementation that + # will not find custom indexes. But it will select by name without passing + # a column. + # + # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: + # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 + # + # Or this can be removed when we no longer support postgres < 9.5, so we + # can use `CREATE INDEX IF NOT EXISTS`. def index_exists_by_name?(table, index) - indexes(table).map(&:name).include?(index) + # We can't fall back to the normal `index_exists?` method because that + # does not find indexes without passing a column name. + if indexes(table).map(&:name).include?(index.to_s) + true + elsif Gitlab::Database.postgresql? + postgres_exists_by_name?(table, index) + else + false + end + end + + def postgres_exists_by_name?(table, name) + index_sql = <<~SQL + SELECT COUNT(*) + FROM pg_index + JOIN pg_class i ON (indexrelid=i.oid) + JOIN pg_class t ON (indrelid=t.oid) + WHERE i.relname = '#{name}' AND t.relname = '#{table}' + SQL + + connection.select_value(index_sql).to_i > 0 end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index fd4a8832ec2..62d4d0a92a6 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -74,7 +74,7 @@ module Gitlab }.freeze def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] + Gitlab.config.repositories.storages[repository_storage].legacy_disk_path end # Overridden to have the correct `source_type` for the `route` relation diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0fb71976883..5fdd5dcd374 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,8 +2,8 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze - EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze + CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze PLEASE_READ_THIS_BANNER = %Q{ @@ -11,57 +11,81 @@ module Gitlab ===================== PLEASE READ THIS ===================== ============================================================ }.freeze + STAY_STRONG_LINK_TO_DOCS = %Q{ + Stay 💪! For more information, see + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + }.freeze THANKS_FOR_READING_BANNER = %Q{ ============================================================ ==================== THANKS FOR READING ==================== ============================================================\n }.freeze - attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found + attr_reader :ee_repo_dir, :patches_dir + attr_reader :ce_project_url, :ee_repo_url + attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found attr_reader :job_id, :failed_files - def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil) + def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @ce_project_url = ce_project_url - @ce_repo_url = "#{ce_project_url}.git" + @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee') @job_id = job_id end def check ensure_patches_dir - add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") - generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') + # We're generating the patch against the canonical-ce remote since forks' + # master branch are not necessarily up-to-date. + add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") - add_remote('canonical-ee', EE_REPO_URL) + ee_remotes.each do |key, url| + add_remote(key, url) + end + fetch(branch: 'master', depth: 20, remote: 'canonical-ee') status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! - step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) - generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') + step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee') ee_branch_compat_check! end delete_ee_branches_locally! - if status.nil? - true - else - false - end + status.nil? end end private + def fork? + ce_project_url != CANONICAL_CE_PROJECT_URL + end + + def ee_remotes + return @ee_remotes if defined?(@ee_remotes) + + remotes = + { + 'ee' => ee_repo_url, + 'canonical-ee' => CANONICAL_EE_REPO_URL + } + remotes.delete('ee') unless fork? + + @ee_remotes = remotes + end + def add_remote(name, url) step( "Adding the #{name} remote (#{url})", @@ -70,28 +94,32 @@ module Gitlab end def ensure_ee_repo - if Dir.exist?(ee_repo_dir) - step("#{ee_repo_dir} already exists") - else - step( - "Cloning #{EE_REPO_URL} into #{ee_repo_dir}", - %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}] - ) + unless clone_repo(ee_repo_url, ee_repo_dir) + # Fallback to using the canonical EE if there is no forked EE + clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir) end end + def clone_repo(url, dir) + _, status = step( + "Cloning #{url} into #{dir}", + %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}] + ) + status.zero? + end + def ensure_patches_dir FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch:, patch_path:, remote:) + def generate_patch(branch:, patch_path:, branch_remote:, master_remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch, master_remote: remote) + find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote) step( - "Generating the patch against #{remote}/master in #{patch_path}", - %W[git diff --binary #{remote}/master...origin/#{branch}] + "Generating the patch against #{master_remote}/master in #{patch_path}", + %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -109,23 +137,22 @@ module Gitlab end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) - - if status.zero? - @ee_branch_found = ee_branch_prefix - return + ee_remotes.keys.each do |remote| + [ee_branch_prefix, ee_branch_suffix].each do |branch| + _, status = step("Fetching #{remote}/#{ee_branch_prefix}", %W[git fetch #{remote} #{branch}]) + + if status.zero? + @ee_remote_with_branch = remote + @ee_branch_found = branch + return true + end + end end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) - - if status.zero? - @ee_branch_found = ee_branch_suffix - else - puts - puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - throw(:halt_check, :ko) - end + throw(:halt_check, :ko) end def ee_branch_compat_check! @@ -181,10 +208,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found?(master_remote:, branch:) + def merge_base_found?(branch:, branch_remote:, master_remote:) step( "Finding merge base with #{master_remote}/master", - %W[git merge-base #{master_remote}/master origin/#{branch}] + %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -193,7 +220,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:, master_remote:) + def find_merge_base_with_master(branch:, branch_remote:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -202,10 +229,10 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: branch, depth: depth, remote: branch_remote) fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found?(master_remote: master_remote, branch: branch) + merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote) end raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success @@ -274,6 +301,13 @@ module Gitlab Gitlab::Popen.popen(cmd) end + # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains + # redacted credentials (e.g. "***:****") which are useless in instructions + # the job gives. + def ce_public_repo_url + "#{ce_project_url}.git" + end + def applies_cleanly_msg(branch) %Q{ #{PLEASE_READ_THIS_BANNER} @@ -288,13 +322,15 @@ module Gitlab end def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + ee_repos = ee_remotes.values.uniq + %Q{ #{PLEASE_READ_THIS_BANNER} 💥 Oh no! 💥 The `#{ce_branch}` branch does not apply cleanly to the current EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch - was found in the EE repository. + was found in #{ee_repos.join(' nor in ')}. If you're a community contributor, don't worry, someone from GitLab Inc. will take care of this, and you don't have to do anything. @@ -314,17 +350,17 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD - $ git fetch #{ce_repo_url} #{ce_branch} + $ git fetch #{ce_public_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick - You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. + Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. 2. Apply your branch's patch to EE # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ wget #{patch_url} && git apply --3way #{ce_patch_name} @@ -356,10 +392,9 @@ module Gitlab ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. - Once this is done, you can retry this failed build, and it should pass. + Once this is done, you can retry this failed job, and it should pass. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end @@ -371,16 +406,15 @@ module Gitlab The `#{ce_branch}` does not apply cleanly to the current EE/master, and even though a `#{ee_branch_found}` branch - exists in the EE repository, it does not apply cleanly either to + exists in #{ee_repo_url}, it does not apply cleanly either to EE/master! #{conflicting_files_msg} Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and - retry this build. + retry this job. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 6659efa0961..0b8f6cfe3cb 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -90,7 +90,7 @@ module Gitlab end def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "".encode("UTF-16BE")) .encode("UTF-8") .gsub("\0".encode("UTF-8"), "") end diff --git a/lib/gitlab/git/checksum.rb b/lib/gitlab/git/checksum.rb new file mode 100644 index 00000000000..3ef0f0a8854 --- /dev/null +++ b/lib/gitlab/git/checksum.rb @@ -0,0 +1,82 @@ +module Gitlab + module Git + class Checksum + include Gitlab::Git::Popen + + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze + + Failure = Class.new(StandardError) + + attr_reader :path, :relative_path, :storage, :storage_path + + def initialize(storage, relative_path) + @storage = storage + @storage_path = Gitlab.config.repositories.storages[storage].legacy_disk_path + @relative_path = "#{relative_path}.git" + @path = File.join(storage_path, @relative_path) + end + + def calculate + unless repository_exists? + failure!(Gitlab::Git::Repository::NoRepository, 'No repository for such path') + end + + calculate_checksum_by_shelling_out + end + + private + + def repository_exists? + raw_repository.exists? + end + + def calculate_checksum_by_shelling_out + args = %W(--git-dir=#{path} show-ref --heads --tags) + output, status = run_git(args) + + if status&.zero? + refs = output.split("\n") + + result = refs.inject(nil) do |checksum, ref| + value = Digest::SHA1.hexdigest(ref).hex + + if checksum.nil? + value + else + checksum ^ value + end + end + + result.to_s(16) + else + # Empty repositories return with a non-zero status and an empty output. + if output&.empty? + EMPTY_REPOSITORY_CHECKSUM + else + failure!(Gitlab::Git::Checksum::Failure, output) + end + end + end + + def failure!(klass, message) + Gitlab::GitLogger.error("'git show-ref --heads --tags' in #{path}: #{message}") + + raise klass.new("Could not calculate the checksum for #{path}: #{message}") + end + + def circuit_breaker + @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) + end + + def raw_repository + Gitlab::Git::Repository.new(storage, relative_path, nil) + end + + def run_git(args) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path) + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index 2a9cf10a068..f08dab59ce4 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,17 +2,19 @@ module Gitlab module Git module Conflict class File + UnsupportedEncoding = Class.new(StandardError) + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid - attr_accessor :content + attr_accessor :raw_content - def initialize(repository, commit_oid, conflict, content) + def initialize(repository, commit_oid, conflict, raw_content) @repository = repository @commit_oid = commit_oid @their_path = conflict[:theirs][:path] @our_path = conflict[:ours][:path] @our_mode = conflict[:ours][:mode] - @content = content + @raw_content = raw_content end def lines @@ -29,6 +31,14 @@ module Gitlab end end + def content + @content ||= @raw_content.dup.force_encoding('UTF-8') + + raise UnsupportedEncoding unless @content.valid_encoding? + + @content + end + def type lines unless @type diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index 3effa9d2d31..fb5717dd556 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -4,7 +4,6 @@ module Gitlab class Parser UnresolvableError = Class.new(StandardError) UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) # Recoverable errors - the conflict can be resolved in an editor, but not with # sections. @@ -75,10 +74,6 @@ module Gitlab def validate_text!(text) raise UnmergeableFile if text.blank? # Typically a binary file raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? end def validate_delimiter!(condition) diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index a142ed6b2ef..099709620b3 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -4,20 +4,14 @@ module Gitlab include Gitlab::Git::Popen include Gitlab::Utils::StrongMemoize - ShardNameNotFoundError = Class.new(StandardError) - - # Absolute path to directory where repositories are stored. - # Example: /home/git/repositories - attr_reader :shard_path + # Name of shard where repositories are stored. + # Example: nfs-file06 + attr_reader :shard_name # Relative path is a directory name for repository with .git at the end. # Example: gitlab-org/gitlab-test.git attr_reader :repository_relative_path - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - attr_reader :repository_absolute_path - # This is the path at which the gitlab-shell hooks directory can be found. # It's essential for integration between git and GitLab proper. All new # repositories should have their hooks directory symlinked here. @@ -25,13 +19,12 @@ module Gitlab attr_reader :logger - def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:) - @shard_path = shard_path + def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) + @shard_name = shard_name @repository_relative_path = repository_relative_path @logger = logger @global_hooks_path = global_hooks_path - @repository_absolute_path = File.join(shard_path, repository_relative_path) @output = StringIO.new end @@ -41,6 +34,22 @@ module Gitlab io.read end + # Absolute path to the repository. + # Example: /home/git/repositorities/gitlab-org/gitlab-test.git + # Probably will be removed when we fully migrate to Gitaly, part of + # https://gitlab.com/gitlab-org/gitaly/issues/1124. + def repository_absolute_path + strong_memoize(:repository_absolute_path) do + File.join(shard_path, repository_relative_path) + end + end + + def shard_path + strong_memoize(:shard_path) do + Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path + end + end + # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) @@ -53,12 +62,12 @@ module Gitlab end end - def fork_repository(new_shard_path, new_repository_relative_path) + def fork_repository(new_shard_name, new_repository_relative_path) Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| if is_enabled - gitaly_fork_repository(new_shard_path, new_repository_relative_path) + gitaly_fork_repository(new_shard_name, new_repository_relative_path) else - git_fork_repository(new_shard_path, new_repository_relative_path) + git_fork_repository(new_shard_name, new_repository_relative_path) end end end @@ -205,17 +214,6 @@ module Gitlab private - def shard_name - strong_memoize(:shard_name) do - shard_name_from_shard_path(shard_path) - end - end - - def shard_name_from_shard_path(shard_path) - Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || - raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") - end - def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) @@ -252,8 +250,9 @@ module Gitlab false end - def git_fork_repository(new_shard_path, new_repository_relative_path) + def git_fork_repository(new_shard_name, new_repository_relative_path) from_path = repository_absolute_path + new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path to_path = File.join(new_shard_path, new_repository_relative_path) # The repository cannot already exist @@ -271,8 +270,8 @@ module Gitlab run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end - def gitaly_fork_repository(new_shard_path, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + def gitaly_fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 4a43b9b444d..4b505312f60 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -46,6 +46,8 @@ module Gitlab iterator = State.new @content.split("\n").each_with_object(iterator) do |text, iterator| + text.chomp! + next if text =~ /^\s*#/ if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ @@ -55,7 +57,7 @@ module Gitlab next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ - value = $~[:value].chomp + value = $~[:value] iterator.set_attribute($~[:key], value) end end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/hook_env.rb index 9d0b47a1a6d..455e8451c10 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -3,37 +3,39 @@ module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git - # commands may need. + # commands need during internal API calls made from Git push hooks. # # For example, in pre-receive hooks, new objects are put in a temporary # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved # (this would break push rules for instance). # # This class is thread-safe via RequestStore. - class Env + class HookEnv WHITELISTED_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze - def self.set(env) + def self.set(gl_repository, env) return unless RequestStore.active? - RequestStore.store[:gitlab_git_env] = whitelist_git_env(env) + raise "missing gl_repository" if gl_repository.blank? + + RequestStore.store[:gitlab_git_env] ||= {} + RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end - def self.all + def self.all(gl_repository) return {} unless RequestStore.active? - RequestStore.fetch(:gitlab_git_env) { {} } + h = RequestStore.fetch(:gitlab_git_env) { {} } + h.fetch(gl_repository, {}) end - def self.to_env_hash + def self.to_env_hash(gl_repository) env = {} - all.compact.each do |key, value| + all(gl_repository).compact.each do |key, value| value = value.join(File::PATH_SEPARATOR) if value.is_a?(Array) env[key.to_s] = value end @@ -41,10 +43,6 @@ module Gitlab env end - def self.[](key) - all[key] - end - def self.whitelist_git_env(env) env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 208710b0935..8d97bfb0e6a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,7 @@ module Gitlab class Repository include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen + include Gitlab::EncodingHelper ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ GIT_OBJECT_DIRECTORY @@ -93,9 +94,9 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - storage_path = Gitlab.config.repositories.storages[@storage]['path'] + storage_path = Gitlab.config.repositories.storages[@storage].legacy_disk_path @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage_path, + storage, relative_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger @@ -516,10 +517,6 @@ module Gitlab end end - def sha_from_ref(ref) - rev_parse_target(ref).oid - end - # Return the object that +revspec+ points to. If +revspec+ is an # annotated tag, then return the tag's target instead. def rev_parse_target(revspec) @@ -888,7 +885,8 @@ module Gitlab end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs) do |is_enabled| + gitaly_migrate(:delete_refs, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_delete_refs(*ref_names) else @@ -1483,7 +1481,7 @@ module Gitlab names.lines.each do |line| next unless line.start_with?(refs_prefix) - refs << line.rstrip[left_slice_count..-1] + refs << encode_utf8(line.rstrip[left_slice_count..-1]) end refs @@ -1748,21 +1746,11 @@ module Gitlab end def alternate_object_directories - relative_paths = relative_object_directories - - if relative_paths.any? - relative_paths.map { |d| File.join(path, d) } - else - absolute_object_directories.flat_map { |d| d.split(File::PATH_SEPARATOR) } - end + relative_object_directories.map { |d| File.join(path, d) } end def relative_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def absolute_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).flatten.compact + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact end # Get the content of a blob for a given commit. If the blob is a commit @@ -2409,6 +2397,10 @@ module Gitlab def rev_list_param(spec) spec == :all ? ['--all'] : spec end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end end end end diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb index d3c37f82101..2f611cef37b 100644 --- a/lib/gitlab/git/storage/checker.rb +++ b/lib/gitlab/git/storage/checker.rb @@ -35,7 +35,7 @@ module Gitlab def initialize(storage, logger = Rails.logger) @storage = storage config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] + @storage_path = config.legacy_disk_path @logger = logger @hostname = Gitlab::Environment.hostname diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 898bb1b65be..e35054466ff 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -25,7 +25,7 @@ module Gitlab if !config.present? NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config['path'].present? + elsif !config.legacy_disk_path.present? NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) else new(storage, hostname) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 52b44b9b3c5..8d82820915d 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -29,7 +29,6 @@ module Gitlab @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) - gollum_wiki.clear_cache else gollum_write_page(name, format, content, commit_details) end @@ -40,7 +39,6 @@ module Gitlab @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| if is_enabled gitaly_delete_page(page_path, commit_details) - gollum_wiki.clear_cache else gollum_delete_page(page_path, commit_details) end @@ -51,7 +49,6 @@ module Gitlab @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) - gollum_wiki.clear_cache else gollum_update_page(page_path, title, format, content, commit_details) end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 6400089a22f..ed0644f6cf1 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -53,7 +53,7 @@ module Gitlab ensure_project_on_push!(cmd, changes) check_project_accessibility! - check_project_moved! + add_project_moved_message! check_repository_existence! case cmd @@ -99,8 +99,6 @@ module Gitlab end def check_active_user! - return if deploy_key? - if user && !user_access.allowed? raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] end @@ -125,16 +123,12 @@ module Gitlab end end - def check_project_moved! + def add_project_moved_message! return if redirected_path.nil? project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) - if project_moved.permanent_redirect? - project_moved.add_message - else - raise ProjectMovedError, project_moved.message(rejected: true) - end + project_moved.add_message end def check_command_disabled!(cmd) @@ -219,7 +213,7 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if deploy_key + if deploy_key? unless deploy_key.can_push_to?(project) raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] end @@ -309,8 +303,10 @@ module Gitlab case actor when User actor + when DeployKey + nil when Key - actor.user unless actor.is_a?(DeployKey) + actor.user when :ci nil end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 8ca30ffc232..0abae70c443 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -83,6 +83,10 @@ module Gitlab end end + def self.random_storage + Gitlab.config.repositories.storages.keys.sample + end + def self.address(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index 97c13d1fdb0..c275a065bce 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -17,7 +17,7 @@ module Gitlab current_file = file_from_gitaly_header(gitaly_file.header) else - current_file.content << gitaly_file.content + current_file.raw_content << gitaly_file.content end end end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 58c356edfd1..f2d699d9dfb 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -3,6 +3,17 @@ module Gitlab class RemoteService MAX_MSG_SIZE = 128.kilobytes.freeze + def self.exists?(remote_url) + request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url) + + response = GitalyClient.call(GitalyClient.random_storage, + :remote_service, + :find_remote_repository, request, + timeout: GitalyClient.medium_timeout) + + response.exists + end + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb new file mode 100644 index 00000000000..8668caf0c55 --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -0,0 +1,35 @@ +module Gitlab + module GitalyClient + # This is a chokepoint that is meant to help us stop remove all places + # where production code (app, config, db, lib) touches Git repositories + # directly. + class StorageSettings + DirectPathAccessError = Class.new(StandardError) + + # This class will give easily recognizable NoMethodErrors + Deprecated = Class.new + + attr_reader :legacy_disk_path + + def initialize(storage) + raise "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) + + # Support a nil 'path' field because some of the circuit breaker tests use it. + @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] + + storage['path'] = Deprecated + @hash = storage + end + + def gitaly_address + @hash.fetch(:gitaly_address) + end + + private + + def method_missing(m, *args, &block) + @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index a8c6d478de8..405567db94a 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -3,11 +3,9 @@ module Gitlab module Util class << self def repository(repository_storage, relative_path, gl_repository) - git_object_directory = Gitlab::Git::Env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence || - Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].presence - git_alternate_object_directories = - Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']).presence || - Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']).flat_map { |d| d.split(File::PATH_SEPARATOR) } + git_env = Gitlab::Git::HookEnv.all(gl_repository) + git_object_directory = git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence + git_alternate_object_directories = Array.wrap(git_env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']) Gitaly::Repository.new( storage_name: repository_storage, diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index ab0b751fe24..01168abde6c 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -16,7 +16,8 @@ module Gitlab # Returns true if we should import the wiki for the project. def import_wiki? client.repository(project.import_source)&.has_wiki && - !project.wiki_repository_exists? + !project.wiki_repository_exists? && + Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end # Imports the repository data. @@ -55,10 +56,8 @@ module Gitlab def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" - wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - storage_path = project.repository_storage_path - gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) true rescue Gitlab::Shell::Error => e @@ -70,6 +69,10 @@ module Gitlab end end + def wiki_url + project.import_url.sub(/\.git\z/, '.wiki.git') + end + def update_clone_time project.update_column(:last_repository_updated_at, Time.zone.now) end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index afaa59b1018..6e554383270 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -77,7 +77,7 @@ module Gitlab end def storage_path(storage_name) - storages_paths&.dig(storage_name, 'path') + storages_paths[storage_name]&.legacy_disk_path end # All below test methods use shell commands to perform actions on storage volumes. diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb new file mode 100644 index 00000000000..9aca3b0fb26 --- /dev/null +++ b/lib/gitlab/http.rb @@ -0,0 +1,13 @@ +# This class is used as a proxy for all outbounding http connection +# coming from callbacks, services and hooks. The direct use of the HTTParty +# is discouraged because it can lead to several security problems, like SSRF +# calling internal IP or services. +module Gitlab + class HTTP + BlockedUrlError = Class.new(StandardError) + + include HTTParty # rubocop:disable Gitlab/HTTParty + + connection_adapter ProxyHTTPConnectionAdapter + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb new file mode 100644 index 00000000000..aef371d81eb --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -0,0 +1,83 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class BaseAfterExportStrategy + include ActiveModel::Validations + extend Forwardable + + StrategyError = Class.new(StandardError) + + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + + private + + attr_reader :project, :current_user + + public + + def initialize(attributes = {}) + @options = OpenStruct.new(attributes) + + self.class.instance_eval do + def_delegators :@options, *attributes.keys + end + end + + def execute(current_user, project) + return unless project&.export_project_path + + @project = project + @current_user = current_user + + if invalid? + log_validation_errors + + return + end + + create_or_update_after_export_lock + strategy_execute + + true + rescue => e + project.import_export_shared.error(e) + false + ensure + delete_after_export_lock + end + + def to_json(options = {}) + @options.to_h.merge!(klass: self.class.name).to_json + end + + def self.lock_file_path(project) + return unless project&.export_path + + File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + end + + protected + + def strategy_execute + raise NotImplementedError + end + + private + + def create_or_update_after_export_lock + FileUtils.touch(self.class.lock_file_path(project)) + end + + def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) + end + + def log_validation_errors + errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb new file mode 100644 index 00000000000..4371a7eff56 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class DownloadNotificationStrategy < BaseAfterExportStrategy + private + + def strategy_execute + notification_service.project_exported(project, current_user) + end + + def notification_service + @notification_service ||= NotificationService.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb new file mode 100644 index 00000000000..938664a95a1 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -0,0 +1,61 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class WebUploadStrategy < BaseAfterExportStrategy + PUT_METHOD = 'PUT'.freeze + POST_METHOD = 'POST'.freeze + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + + validates :url, url: true + + validate do + unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase) + errors.add(:http_method, INVALID_HTTP_METHOD) + end + end + + def initialize(url:, http_method: PUT_METHOD) + super + end + + protected + + def strategy_execute + handle_response_error(send_file) + + project.remove_exported_project_file + end + + def handle_response_error(response) + unless response.success? + error_code = response.dig('Error', 'Code') || response.code + error_message = response.dig('Error', 'Message') || response.message + + raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + end + end + + private + + def send_file + export_file = File.open(project.export_project_path) + + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + ensure + export_file.close if export_file + end + + def send_file_options(export_file) + { + body_stream: export_file, + headers: headers + } + end + + def headers + { 'Content-Length' => File.size(project.export_project_path).to_s } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb new file mode 100644 index 00000000000..7eabcae2380 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class AfterExportStrategyBuilder + StrategyNotFoundError = Class.new(StandardError) + + def self.build!(strategy_klass, attributes = {}) + return default_strategy.new unless strategy_klass + + attributes ||= {} + klass = strategy_klass.constantize rescue nil + + unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy + raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + end + + klass.new(**attributes.symbolize_keys) + end + + def self.default_strategy + AfterExportStrategies::DownloadNotificationStrategy + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 791a54e1b69..598832fb2df 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -19,7 +19,7 @@ module Gitlab custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 3d3d998a6a3..6d7c36ce38b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -22,7 +22,7 @@ module Gitlab def error(error) error_out(error.message, caller[0].dup) - @errors << error.message + add_error_message(error.message) # Debug: if error.backtrace @@ -32,6 +32,14 @@ module Gitlab end end + def add_error_message(error_message) + @errors << error_message + end + + def after_export_in_progress? + File.exist?(after_export_lock_file) + end + private def relative_path @@ -45,6 +53,10 @@ module Gitlab def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 0526ef9eb13..7edd0ad2033 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -259,7 +259,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index db8bdde74b2..47b4af5d649 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -4,6 +4,8 @@ require 'prometheus/client/rack/exporter' module Gitlab module Metrics class SidekiqMetricsExporter < Daemon + LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') + def enabled? Gitlab::Metrics.metrics_folder_present? && settings.enabled end @@ -17,7 +19,13 @@ module Gitlab attr_reader :server def start_working - @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + logger = WEBrick::Log.new(LOG_FILENAME) + access_log = [ + [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] + ] + + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address, + Logger: logger, AccessLog: access_log) server.mount "/", Rack::Handler::WEBrick, rack_app server.start end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index d9d5f90596f..7f63e39b3aa 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -13,7 +13,7 @@ module Gitlab end def call(env) - ReadOnly::Controller.new(@app, env).call + ::Gitlab::Middleware::ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb new file mode 100644 index 00000000000..35ed3a5ac05 --- /dev/null +++ b/lib/gitlab/omniauth_initializer.rb @@ -0,0 +1,75 @@ +module Gitlab + class OmniauthInitializer + def initialize(devise_config) + @devise_config = devise_config + end + + def execute(providers) + providers.each do |provider| + add_provider(provider['name'].to_sym, *arguments_for(provider)) + end + end + + private + + def add_provider(*args) + @devise_config.omniauth(*args) + end + + def arguments_for(provider) + provider_arguments = [] + + %w[app_id app_secret].each do |argument| + provider_arguments << provider[argument] if provider[argument] + end + + case provider['args'] + when Array + # An Array from the configuration will be expanded. + provider_arguments.concat provider['args'] + when Hash + hash_arguments = provider['args'].merge(provider_defaults(provider)) + + # A Hash from the configuration will be passed as is. + provider_arguments << hash_arguments.symbolize_keys + end + + provider_arguments + end + + def provider_defaults(provider) + case provider['name'] + when 'cas3' + { on_single_sign_out: cas3_signout_handler } + when 'authentiq' + { remote_sign_out_handler: authentiq_signout_handler } + when 'shibboleth' + { fail_with_empty_uid: true } + else + {} + end + end + + def cas3_signout_handler + lambda do |request| + ticket = request.params[:session_index] + raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket) + + Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket) + true + end + end + + def authentiq_signout_handler + lambda do |request| + authentiq_session = request.params['sid'] + if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session) + Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session) + true + else + false + end + end + end + end +end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 6c2b2036074..92a308a12dc 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -5,6 +5,7 @@ module Gitlab def self.enabled?(user = nil) return true if Rails.env.development? + return true if user&.admin? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 98a168b43bb..18540e64d4c 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -92,8 +92,8 @@ module Gitlab if type && time @load_times_by_model ||= {} - @load_times_by_model[type] ||= 0 - @load_times_by_model[type] += time.to_f + @load_times_by_model[type] ||= [] + @load_times_by_model[type] << time.to_f end super @@ -135,8 +135,12 @@ module Gitlab def self.log_load_times_by_model(logger) return unless logger.respond_to?(:load_times_by_model) - logger.load_times_by_model.to_a.sort_by(&:last).reverse.each do |(model, time)| - logger.info("#{model} total: #{time.round(2)}ms") + summarised_load_times = logger.load_times_by_model.to_a.map do |(model, times)| + [model, times.count, times.sum] + end + + summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb new file mode 100644 index 00000000000..d682289b632 --- /dev/null +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -0,0 +1,34 @@ +# This class is part of the Gitlab::HTTP wrapper. Depending on the value +# of the global setting allow_local_requests_from_hooks_and_services this adapter +# will allow/block connection to internal IPs and/or urls. +# +# This functionality can be overriden by providing the setting the option +# allow_local_requests = true in the request. For example: +# Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true) +# +# This option will take precedence over the global setting. +module Gitlab + class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter + def connection + unless allow_local_requests? + begin + Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + end + end + + super + end + + private + + def allow_local_requests? + options.fetch(:allow_local_requests, allow_settings_local_requests?) + end + + def allow_settings_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + end + end +end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 79265cf952d..1fa2a19b0af 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -21,11 +21,11 @@ module Gitlab result = repo_path storage = Gitlab.config.repositories.storages.values.find do |params| - repo_path.start_with?(params['path']) + repo_path.start_with?(params.legacy_disk_path) end if storage - result = result.sub(storage['path'], '') + result = result.sub(storage.legacy_disk_path, '') elsif fail_on_not_found raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 07d7c91cb5d..e5c02dd8ecc 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,7 +24,7 @@ module Gitlab address = val['gitaly_address'] end - storages << { name: key, path: val['path'] } + storages << { name: key, path: val.legacy_disk_path } end if Rails.env.test? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 3a8f5826818..67407b651a5 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -82,7 +82,7 @@ module Gitlab repository.gitaly_repository_client.create_repository true else - repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path) Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) end end @@ -93,12 +93,12 @@ module Gitlab # Import repository # - # storage - project's storage path + # storage - project's storage name # name - project disk path # url - URL to import from # # Ex. - # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") + # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) @@ -131,8 +131,7 @@ module Gitlab if is_enabled repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else - storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) + local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -156,13 +155,13 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage path - # forked_from_disk_path - project disk path - # forked_to_storage - forked-to project's storage path - # forked_to_disk_path - forked project disk path + # forked_from_storage - forked-from project's storage name + # forked_from_disk_path - project disk relative path + # forked_to_storage - forked-to project's storage name + # forked_to_disk_path - forked project disk relative path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") + # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) @@ -420,16 +419,16 @@ module Gitlab private - def gitlab_projects(shard_path, disk_path) + def gitlab_projects(shard_name, disk_path) Gitlab::Git::GitlabProjects.new( - shard_path, + shard_name, disk_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? @@ -442,7 +441,7 @@ module Gitlab end end - cmd = gitlab_projects(storage_path, repository_relative_path) + cmd = gitlab_projects(storage_name, repository_relative_path) success = cmd.fetch_remote(remote, git_timeout, vars) @@ -478,7 +477,7 @@ module Gitlab def gitaly_namespace_client(storage_path) storage, _value = Gitlab.config.repositories.storages.find do |storage, value| - value['path'] == storage_path + value.legacy_disk_path == storage_path end Gitlab::GitalyClient::NamespaceService.new(storage) diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb new file mode 100644 index 00000000000..98f8222fd03 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module SidekiqLogging + class JSONFormatter + def call(severity, timestamp, progname, data) + output = { + severity: severity, + time: timestamp.utc.iso8601(3) + } + + case data + when String + output[:message] = data + when Hash + output.merge!(data) + end + + output.to_json + "\n" + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb new file mode 100644 index 00000000000..9a89ae70b98 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -0,0 +1,96 @@ +module Gitlab + module SidekiqLogging + class StructuredLogger + START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze + DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + + def call(job, queue) + started_at = current_time + base_payload = parse_job(job) + + Sidekiq.logger.info log_job_start(started_at, base_payload) + + yield + + Sidekiq.logger.info log_job_done(started_at, base_payload) + rescue => job_exception + Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + + raise + end + + private + + def base_message(payload) + "#{payload['class']} JID-#{payload['jid']}" + end + + def log_job_start(started_at, payload) + payload['message'] = "#{base_message(payload)}: start" + payload['job_status'] = 'start' + + payload + end + + def log_job_done(started_at, payload, job_exception = nil) + payload = payload.dup + payload['duration'] = elapsed(started_at) + payload['completed_at'] = Time.now.utc + + message = base_message(payload) + + if job_exception + payload['message'] = "#{message}: fail: #{payload['duration']} sec" + payload['job_status'] = 'fail' + payload['error_message'] = job_exception.message + payload['error'] = job_exception.class + payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace) + else + payload['message'] = "#{message}: done: #{payload['duration']} sec" + payload['job_status'] = 'done' + end + + convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) + + payload + end + + def parse_job(job) + job = job.dup + + # Add process id params + job['pid'] = ::Process.pid + + job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + + convert_to_iso8601(job, START_TIMESTAMP_FIELDS) + + job + end + + def convert_to_iso8601(payload, keys) + keys.each do |key| + payload[key] = format_time(payload[key]) if payload[key] + end + end + + def elapsed(start) + (current_time - start).round(3) + end + + def current_time + Gitlab::Metrics::System.monotonic_time + end + + def backtrace_cleaner + @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new + end + + def format_time(timestamp) + return timestamp if timestamp.is_a?(String) + + Time.at(timestamp).utc.iso8601(3) + end + end + end +end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 34bee6fecbe..42be301fd9b 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -129,7 +129,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end @@ -138,7 +138,7 @@ module Gitlab end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end def user_home diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 13150ddab67..db97f65bd54 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -2,49 +2,84 @@ require 'resolv' module Gitlab class UrlBlocker - class << self - # Used to specify what hosts and port numbers should be prohibited for project - # imports. - VALID_PORTS = [22, 80, 443].freeze - - def blocked_url?(url) - return false if url.nil? + BlockedUrlError = Class.new(StandardError) - blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"] - blocked_ips.concat(Socket.ip_address_list.map(&:ip_address)) + class << self + def validate!(url, allow_localhost: false, allow_local_network: true, valid_ports: []) + return true if url.nil? begin uri = Addressable::URI.parse(url) - # Allow imports from the GitLab instance itself but only from the configured ports - return false if internal?(uri) + rescue Addressable::URI::InvalidURIError + raise BlockedUrlError, "URI is invalid" + end - return true if blocked_port?(uri.port) - return true if blocked_user_or_hostname?(uri.user) - return true if blocked_user_or_hostname?(uri.hostname) + # Allow imports from the GitLab instance itself but only from the configured ports + return true if internal?(uri) - server_ips = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM).map(&:ip_address) - return true if (blocked_ips & server_ips).any? - rescue Addressable::URI::InvalidURIError - return true + port = uri.port || uri.default_port + validate_port!(port, valid_ports) if valid_ports.any? + validate_user!(uri.user) + validate_hostname!(uri.hostname) + + begin + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) rescue SocketError - return false + return true end + validate_localhost!(addrs_info) unless allow_localhost + validate_local_network!(addrs_info) unless allow_local_network + + true + end + + def blocked_url?(*args) + validate!(*args) + false + rescue BlockedUrlError + true end private - def blocked_port?(port) - return false if port.blank? + def validate_port!(port, valid_ports) + return if port.blank? + # Only ports under 1024 are restricted + return if port >= 1024 + return if valid_ports.include?(port) + + raise BlockedUrlError, "Only allowed ports are #{valid_ports.join(', ')}, and any over 1024" + end + + def validate_user!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Username needs to start with an alphanumeric character" + end + + def validate_hostname!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + end + + def validate_localhost!(addrs_info) + local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + return if (local_ips & addrs_info.map(&:ip_address)).empty? - port < 1024 && !VALID_PORTS.include?(port) + raise BlockedUrlError, "Requests to localhost are not allowed" end - def blocked_user_or_hostname?(value) - return false if value.blank? + def validate_local_network!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } - value !~ /\A\p{Alnum}/ + raise BlockedUrlError, "Requests to the local network are not allowed" end def internal?(uri) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 37d3512990e..8c0a4d55ea2 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -30,6 +30,7 @@ module Gitlab usage_data end + # rubocop:disable Metrics/AbcSize def system_usage_data { counts: { @@ -50,6 +51,12 @@ module Gitlab clusters: ::Clusters::Cluster.count, clusters_enabled: ::Clusters::Cluster.enabled.count, clusters_disabled: ::Clusters::Cluster.disabled.count, + clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, + clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, + clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, + clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, + clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, + clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index fe51edbdeeb..970e2a7b718 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - LfsObject.all + LfsObject.with_files_stored_locally end def expected_checksum(lfs_object) diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 6972e517ea5..0ffa71a6d72 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - Upload.all + Upload.with_files_stored_locally end def expected_checksum(upload) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0b0d667d4fd..b102812ec12 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -21,29 +21,18 @@ module Gitlab raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) project = repository.project - repo_path = repository.path_to_repo - params = { + + { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, - RepoPath: repo_path, - ShowAllRefs: show_all_refs - } - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - params[:GitalyServer] = server - - params - end - - def lfs_upload_ok(oid, size) - { - StoreLFSPath: LfsObjectUploader.workhorse_upload_path, - LfsOid: oid, - LfsSize: size + ShowAllRefs: show_all_refs, + Repository: repository.gitaly_repository.to_h, + RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitalyServer: { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } } end @@ -52,7 +41,7 @@ module Gitlab end def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'GetBlobRequest' => { @@ -80,7 +69,7 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? - if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) params.merge!( 'GitalyServer' => gitaly_server_hash(repository), 'GitalyRepository' => repository.gitaly_repository.to_h @@ -97,7 +86,7 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawDiffRequest' => Gitaly::RawDiffRequest.new( @@ -115,7 +104,7 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawPatchRequest' => Gitaly::RawPatchRequest.new( diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 65ccdb3c347..85f78e44f32 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -22,16 +22,14 @@ module Mattermost # going. class Session include Doorkeeper::Helpers::Controller - include HTTParty LEASE_TIMEOUT = 60 - base_uri Settings.mattermost.host - - attr_accessor :current_resource_owner, :token + attr_accessor :current_resource_owner, :token, :base_uri def initialize(current_user) @current_resource_owner = current_user + @base_uri = Settings.mattermost.host end def with_session @@ -73,24 +71,32 @@ module Mattermost def get(path, options = {}) handle_exceptions do - self.class.get(path, options.merge(headers: @headers)) + Gitlab::HTTP.get(path, build_options(options)) end end def post(path, options = {}) handle_exceptions do - self.class.post(path, options.merge(headers: @headers)) + Gitlab::HTTP.post(path, build_options(options)) end end def delete(path, options = {}) handle_exceptions do - self.class.delete(path, options.merge(headers: @headers)) + Gitlab::HTTP.delete(path, build_options(options)) end end private + def build_options(options) + options.tap do |hash| + hash[:headers] = @headers + hash[:allow_local_requests] = true + hash[:base_uri] = base_uri if base_uri.presence + end + end + def create raise Mattermost::NoSessionError unless oauth_uri raise Mattermost::NoSessionError unless token_uri @@ -165,14 +171,14 @@ module Mattermost def handle_exceptions yield - rescue HTTParty::Error => e + rescue Gitlab::HTTP::Error => e raise Mattermost::ConnectionError.new(e.message) rescue Errno::ECONNREFUSED => e raise Mattermost::ConnectionError.new(e.message) end def parse_cookie(response) - cookie_hash = CookieHash.new + cookie_hash = Gitlab::HTTP::CookieHash.new response.get_fields('Set-Cookie').each { |c| cookie_hash.add_cookies(c) } cookie_hash end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index 3bef68a1bcb..c08d3e933a8 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -9,14 +9,15 @@ module MicrosoftTeams result = false begin - response = HTTParty.post( + response = Gitlab::HTTP.post( @webhook.to_str, headers: @header, + allow_local_requests: true, body: body(options) ) result = true if response - rescue HTTParty::Error, StandardError => error + rescue Gitlab::HTTP::Error, StandardError => error Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") end diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb index b8446300f72..b5f443abe06 100644 --- a/lib/system_check/orphans/namespace_check.rb +++ b/lib/system_check/orphans/namespace_check.rb @@ -6,8 +6,8 @@ module SystemCheck def multi_check Gitlab.config.repositories.storages.each do |storage_name, repository_storage| $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) + $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) orphans = (toplevel_namespace_dirs - existing_namespaces) print_orphans(orphans, storage_name) diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb index 9b6b2429783..5ef0b93ad08 100644 --- a/lib/system_check/orphans/repository_check.rb +++ b/lib/system_check/orphans/repository_check.rb @@ -6,10 +6,12 @@ module SystemCheck def multi_check Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + storage_path = repository_storage.legacy_disk_path + $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) - repositories = disk_repositories(repository_storage['path']) + repositories = disk_repositories(storage_path) orphans = (repositories - fetch_repositories(storage_name)) print_orphans(orphans, storage_name) diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake new file mode 100644 index 00000000000..bfca4bfb3f7 --- /dev/null +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -0,0 +1,25 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Migrate files for artifacts to comply with new storage format" +namespace :gitlab do + namespace :artifacts do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts') + + Ci::Build.joins(:project) + .with_artifacts_stored_locally + .find_each(batch_size: 10) do |build| + begin + build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) + build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + + logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + rescue => e + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2403f57f05a..abef8cd2bcc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -61,7 +61,7 @@ namespace :gitlab do puts "Repo base directory exists?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " if File.exist?(repo_base_path) @@ -86,7 +86,7 @@ namespace :gitlab do puts "Repo storage directories are symlinks?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -110,7 +110,7 @@ namespace :gitlab do puts "Repo paths access is drwxrws---?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -140,7 +140,7 @@ namespace :gitlab do puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 2453079911d..d6d15285489 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -12,7 +12,7 @@ namespace :gitlab do namespaces = Namespace.pluck(:path) namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = repository_storage['path'] + git_base_path = repository_storage.legacy_disk_path all_dirs = Dir.glob(git_base_path + '/*') puts git_base_path.color(:yellow) @@ -54,7 +54,7 @@ namespace :gitlab do move_suffix = "+orphaned+#{Time.now.to_i}" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = repository_storage['path'] + repo_root = repository_storage.legacy_disk_path # Look for global repos (legacy, depth 1) and normal repos (depth 2) IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 45e9a1a1c72..47ed522aec3 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -68,7 +68,7 @@ namespace :gitlab do puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage['path']}" + puts "- #{name}: \t#{repository_storage.legacy_disk_path}" end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake new file mode 100644 index 00000000000..a45e5ca91e0 --- /dev/null +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -0,0 +1,22 @@ +require 'logger' + +desc "GitLab | Migrate LFS objects to remote storage" +namespace :gitlab do + namespace :lfs do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to object storage') + + LfsObject.with_files_stored_locally + .find_each(batch_size: 10) do |lfs_object| + begin + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 7728c485e8d..6b22499a5c8 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :two_factor do desc "GitLab | Disable Two-factor authentication (2FA) for all users" - task disable_for_all_users: :environment do + task disable_for_all_users: :gitlab_environment do scope = User.with_two_factor count = scope.count diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake new file mode 100644 index 00000000000..78e18992a8e --- /dev/null +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -0,0 +1,34 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| + batch_size = ENV.fetch('BATCH', 200).to_i + @to_store = ObjectStorage::Store::REMOTE + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + end + + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, + @mounted_as, + @to_store) + puts "Enqueued job ##{index}: #{job}" + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + end + + def uploads + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where(store: [nil, ObjectStorage::Store::LOCAL], + uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 1c7a8a90f5c..af30ecb0e9b 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -7,8 +7,8 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') - require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') + require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,6 +17,6 @@ task setup_postgresql: :environment do AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up - ReworkRedirectRoutesIndexes.new.up UsersNameLowerIndex.new.up + AddPathIndexToRedirectRoutes.new.up end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 3e01f91d32c..b52af81fc16 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -4,8 +4,3 @@ desc "GitLab | Run all tests" task :test do Rake::Task["gitlab:test"].invoke end - -unless Rails.env.production? - desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec] -end |