diff options
| author | Tiago Botelho <tiagonbotelho@hotmail.com> | 2018-01-25 09:47:36 +0000 |
|---|---|---|
| committer | Tiago Botelho <tiagonbotelho@hotmail.com> | 2018-01-25 10:00:17 +0000 |
| commit | 4903ff93971d28723ed9d81f7dae2ad2a1d5586a (patch) | |
| tree | 8f6f5cbc302f0820c1ffd0c85528388fd33bb8d7 /lib | |
| parent | 3805cd7ac5e1c9b286bfba667ef45972eab4d084 (diff) | |
| parent | 944c1eb684fe979339262ba6e9f7bf9e1b77fc81 (diff) | |
| download | gitlab-ce-13931-custom-emoji-implementation.tar.gz | |
Merge branch 'master' into 13931-custom-emoji-implementation13931-custom-emoji-implementation
Conflicts:
app/assets/javascripts/dispatcher.js
app/assets/javascripts/pages/projects/shared/project_avatar.js
Diffstat (limited to 'lib')
155 files changed, 2264 insertions, 649 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8094597d238..f3f64244589 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -13,7 +13,8 @@ module API formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new + GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::UserLogger.new ] allow_access_with_scope :api @@ -105,6 +106,7 @@ module API # Keep in alphabetical order mount ::API::AccessRequests + mount ::API::Applications mount ::API::AwardEmoji mount ::API::Boards mount ::API::Branches @@ -119,6 +121,7 @@ module API mount ::API::Features mount ::API::Files mount ::API::Groups + mount ::API::GroupMilestones mount ::API::Internal mount ::API::Issues mount ::API::Jobs @@ -129,8 +132,6 @@ module API mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::ProjectMilestones - mount ::API::GroupMilestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -139,6 +140,7 @@ module API mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects + mount ::API::ProjectMilestones mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/applications.rb b/lib/api/applications.rb new file mode 100644 index 00000000000..b122cdefe4e --- /dev/null +++ b/lib/api/applications.rb @@ -0,0 +1,27 @@ +module API + # External applications API + class Applications < Grape::API + before { authenticated_as_admin! } + + resource :applications do + desc 'Create a new application' do + detail 'This feature was introduced in GitLab 10.5' + success Entities::ApplicationWithSecret + end + params do + requires :name, type: String, desc: 'Application name' + requires :redirect_uri, type: String, desc: 'Application redirect URI' + requires :scopes, type: String, desc: 'Application scopes' + end + post do + application = Doorkeeper::Application.new(declared_params) + + if application.save + present application, with: Entities::ApplicationWithSecret + else + render_validation_error! application + end + end + end + end +end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 366b0dc9a6f..6c706b2b4e1 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,45 +1,46 @@ module API class Boards < Grape::API + include BoardsResponses include PaginationParams before { authenticate! } + helpers do + def board_parent + user_project + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Get all project boards' do - detail 'This feature was introduced in 8.13' - success Entities::Board - end - params do - use :pagination - end - get ':id/boards' do - authorize!(:read_board, user_project) - present paginate(user_project.boards), with: Entities::Board + segment ':id/boards' do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + params do + use :pagination + end + get '/' do + authorize!(:read_board, user_project) + present paginate(board_parent.boards), with: Entities::Board + end + + desc 'Find a project board' do + detail 'This feature was introduced in 10.4' + success Entities::Board + end + get '/:board_id' do + present board, with: Entities::Board + end end params do requires :board_id, type: Integer, desc: 'The ID of a board' end segment ':id/boards/:board_id' do - helpers do - def project_board - board = user_project.boards.first - - if params[:board_id] == board.id - board - else - not_found!('Board') - end - end - - def board_lists - project_board.lists.destroyable - end - end - desc 'Get the lists of a project board' do detail 'Does not include `done` list. This feature was introduced in 8.13' success Entities::List @@ -72,22 +73,13 @@ module API requires :label_id, type: Integer, desc: 'The ID of an existing label' end post '/lists' do - unless available_labels.exists?(params[:label_id]) + unless available_labels_for(user_project).exists?(params[:label_id]) render_api_error!({ error: 'Label not found!' }, 400) end authorize!(:admin_list, user_project) - service = ::Boards::Lists::CreateService.new(user_project, current_user, - { label_id: params[:label_id] }) - - list = service.execute(project_board) - - if list.valid? - present list, with: Entities::List - else - render_validation_error!(list) - end + create_list end desc 'Moves a board list to a new position' do @@ -99,18 +91,11 @@ module API requires :position, type: Integer, desc: 'The position of the list' end put '/lists/:list_id' do - list = project_board.lists.movable.find(params[:list_id]) + list = board_lists.find(params[:list_id]) authorize!(:admin_list, user_project) - service = ::Boards::Lists::MoveService.new(user_project, current_user, - { position: params[:position] }) - - if service.execute(list) - present list, with: Entities::List - else - render_api_error!({ error: "List could not be moved!" }, 400) - end + move_list(list) end desc 'Delete a board list' do @@ -124,12 +109,7 @@ module API authorize!(:admin_list, user_project) list = board_lists.find(params[:list_id]) - destroy_conditionally!(list) do |list| - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) - end - end + destroy_list(list) end end end diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb new file mode 100644 index 00000000000..ead0943a74d --- /dev/null +++ b/lib/api/boards_responses.rb @@ -0,0 +1,50 @@ +module API + module BoardsResponses + extend ActiveSupport::Concern + + included do + helpers do + def board + board_parent.boards.find(params[:board_id]) + end + + def board_lists + board.lists.destroyable + end + + def create_list + create_list_service = + ::Boards::Lists::CreateService.new(board_parent, current_user, { label_id: params[:label_id] }) + + list = create_list_service.execute(board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + def move_list(list) + move_list_service = + ::Boards::Lists::MoveService.new(board_parent, current_user, { position: params[:position].to_i }) + + if move_list_service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + def destroy_list(list) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(board_parent, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end + end +end diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index 598c76f6168..c13154dc0ec 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -17,11 +17,11 @@ module API end def storage_health - @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages + @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages end end - desc 'Get all failing git storages' do + desc 'Get all git storages' do detail 'This feature was introduced in GitLab 9.5' success Entities::RepositoryStorageHealth end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 38e05074353..d8fd6a6eb06 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -82,13 +82,14 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end get ':id/repository/commits/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present commit, with: Entities::CommitDetail + present commit, with: Entities::CommitDetail, stats: params[:stats] end desc 'Get the diff for a specific commit of a project' do diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 281269b1190..b0b7b50998f 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -4,6 +4,16 @@ module API before { authenticate! } + helpers do + def add_deploy_keys_project(project, attrs = {}) + project.deploy_keys_projects.create(attrs) + end + + def find_by_deploy_key(project, key_id) + project.deploy_keys_projects.find_by!(deploy_key: key_id) + end + end + desc 'Return all deploy keys' params do use :pagination @@ -21,28 +31,31 @@ module API before { authorize_admin_project } desc "Get a specific project's deploy keys" do - success Entities::SSHKey + success Entities::DeployKeysProject end params do use :pagination end get ":id/deploy_keys" do - present paginate(user_project.deploy_keys), with: Entities::SSHKey + keys = user_project.deploy_keys_projects.preload(:deploy_key) + + present paginate(keys), with: Entities::DeployKeysProject end desc 'Get single deploy key' do - success Entities::SSHKey + success Entities::DeployKeysProject end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end get ":id/deploy_keys/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - present key, with: Entities::SSHKey + key = find_by_deploy_key(user_project, params[:key_id]) + + present key, with: Entities::DeployKeysProject end desc 'Add new deploy key to currently authenticated user' do - success Entities::SSHKey + success Entities::DeployKeysProject end params do requires :key, type: String, desc: 'The new deploy key' @@ -53,24 +66,31 @@ module API params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys.find_by(key: params[:key]) + key = user_project.deploy_keys_projects + .joins(:deploy_key) + .find_by(keys: { key: params[:key] }) + if key - present key, with: Entities::SSHKey + present key, with: Entities::DeployKeysProject break end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key - user_project.deploy_keys << key - present key, with: Entities::SSHKey + added_key = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) + + present added_key, with: Entities::DeployKeysProject break end # Create a new deploy key - key = DeployKey.new(declared_params(include_missing: false)) - if key.valid? && user_project.deploy_keys << key - present key, with: Entities::SSHKey + key_attributes = { can_push: !!params[:can_push], + deploy_key_attributes: declared_params.except(:can_push) } + key = add_deploy_keys_project(user_project, key_attributes) + + if key.valid? + present key, with: Entities::DeployKeysProject else render_validation_error!(key) end @@ -86,14 +106,21 @@ module API at_least_one_of :title, :can_push end put ":id/deploy_keys/:key_id" do - key = DeployKey.find(params.delete(:key_id)) + deploy_keys_project = find_by_deploy_key(user_project, params[:key_id]) - authorize!(:update_deploy_key, key) + authorize!(:update_deploy_key, deploy_keys_project.deploy_key) - if key.update_attributes(declared_params(include_missing: false)) - present key, with: Entities::SSHKey + can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push] + title = params[:title] || deploy_keys_project.deploy_key.title + + result = deploy_keys_project.update_attributes(can_push: can_push, + deploy_key_attributes: { id: params[:key_id], + title: title }) + + if result + present deploy_keys_project, with: Entities::DeployKeysProject else - render_validation_error!(key) + render_validation_error!(deploy_keys_project) end end @@ -122,7 +149,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the deploy key' end delete ":id/deploy_keys/:key_id" do - key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + key = user_project.deploy_keys.find(params[:key_id]) not_found!('Deploy Key') unless key destroy_conditionally!(key) diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 1efee9a1324..184fae0eb76 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -15,11 +15,13 @@ module API end params do use :pagination + optional :order_by, type: String, values: %w[id iid created_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `ref`' + optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' end get ':id/deployments' do authorize! :read_deployment, user_project - present paginate(user_project.deployments), with: Entities::Deployment + present paginate(user_project.deployments.order(params[:order_by] => params[:sort])), with: Entities::Deployment end desc 'Gets a specific deployment' do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4ad4a1f7867..5b470bd3479 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -65,12 +65,12 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :merge_requests_events, :repository_update_events expose :enable_ssl_verification end class ProjectHook < Hook - expose :project_id, :issues_events, :merge_requests_events + expose :project_id, :issues_events expose :note_events, :pipeline_events, :wiki_page_events expose :job_events end @@ -278,7 +278,7 @@ module API end class CommitDetail < Commit - expose :stats, using: Entities::CommitStats + expose :stats, using: Entities::CommitStats, if: :stats expose :status expose :last_pipeline, using: 'API::Entities::PipelineBasic' end @@ -507,7 +507,16 @@ module API expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds - expose :merge_status + + # Ideally we should deprecate `MergeRequest#merge_status` exposure and + # use `MergeRequest#mergeable?` instead (boolean). + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/42344 for more + # information. + expose :merge_status do |merge_request| + # In order to avoid having a breaking change for users, we keep returning the + # expected values from MergeRequest#merge_status state machine. + merge_request.mergeable? ? 'can_be_merged' : 'cannot_be_merged' + end expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :user_notes_count @@ -554,13 +563,18 @@ module API end class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at, :can_push + expose :id, :title, :key, :created_at end class SSHKeyWithUser < SSHKey expose :user, using: Entities::UserPublic end + class DeployKeysProject < Grape::Entity + expose :deploy_key, merge: true, using: Entities::SSHKey + expose :can_push + end + class GPGKey < Grape::Entity expose :id, :key, :created_at end @@ -714,10 +728,7 @@ module API expose :job_events # Expose serialized properties expose :properties do |service, options| - field_names = service.fields - .select { |field| options[:include_passwords] || field[:type] != 'password' } - .map { |field| field[:name] } - service.properties.slice(*field_names) + service.properties.slice(*service.api_field_names) end end @@ -791,6 +802,8 @@ module API class Board < Grape::Entity expose :id + expose :project, using: Entities::BasicProjectDetails + expose :lists, using: Entities::List do |board| board.lists.destroyable end @@ -862,6 +875,8 @@ module API expose :active expose :is_shared expose :name + expose :online?, as: :online + expose :status end class RunnerDetails < Runner @@ -914,7 +929,7 @@ module API class Trigger < Grape::Entity expose :id expose :token, :description - expose :created_at, :updated_at, :deleted_at, :last_used + expose :created_at, :updated_at, :last_used expose :owner, using: Entities::UserBasic end @@ -1133,6 +1148,7 @@ module API class PagesDomainBasic < Grape::Entity expose :domain expose :url + expose :project_id expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, @@ -1150,5 +1166,15 @@ module API pages_domain end end + + class Application < Grape::Entity + expose :uid, as: :application_id + expose :redirect_uri, as: :callback_url + end + + # Use with care, this exposes the secret + class ApplicationWithSecret < Application + expose :secret + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8ad4b2ecbf3..6134ad2bfc7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -5,6 +5,7 @@ module API SUDO_HEADER = "HTTP_SUDO".freeze SUDO_PARAM = :sudo + API_USER_ENV = 'gitlab.api.user'.freeze def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) @@ -25,6 +26,7 @@ module API check_unmodified_since!(last_updated) status 204 + if block_given? yield resource else @@ -48,10 +50,16 @@ module API validate_access_token!(scopes: scopes_registered_for_endpoint) unless sudo? + save_current_user_in_env(@current_user) if @current_user + @current_user end # rubocop:enable Gitlab/ModuleWithInstanceVariables + def save_current_user_in_env(user) + env[API_USER_ENV] = { user_id: user.id, username: user.username } + end + def sudo? initial_current_user != current_user end @@ -74,8 +82,15 @@ module API page || not_found!('Wiki Page') end - def available_labels - @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + def available_labels_for(label_parent) + search_params = + if label_parent.is_a?(Project) + { project_id: label_parent.id } + else + { group_id: label_parent.id, only_group_labels: true } + end + + LabelsFinder.new(current_user, search_params).execute end def find_user(id) @@ -141,7 +156,9 @@ module API end def find_project_label(id) - label = available_labels.find_by_id(id) || available_labels.find_by_title(id) + labels = available_labels_for(user_project) + label = labels.find_by_id(id) || labels.find_by_title(id) + label || not_found!('Label') end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 322624c6092..9993caa5249 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -3,8 +3,10 @@ module API module CommonHelpers def convert_parameters_from_legacy_format(params) params.tap do |params| - if params[:assignee_id].present? - params[:assignee_ids] = [params.delete(:assignee_id)] + assignee_id = params.delete(:assignee_id) + + if assignee_id.present? + params[:assignee_ids] = [assignee_id] end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eff1c5b70ea..eb67de81a0d 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,11 +1,6 @@ module API module Helpers module InternalHelpers - SSH_GITALY_FEATURES = { - 'git-receive-pack' => [:ssh_receive_pack, Gitlab::GitalyClient::MigrationStatus::OPT_IN], - 'git-upload-pack' => [:ssh_upload_pack, Gitlab::GitalyClient::MigrationStatus::OPT_OUT] - }.freeze - attr_reader :redirected_path def wiki? @@ -102,8 +97,14 @@ module API # Return the Gitaly Address if it is enabled def gitaly_payload(action) - feature, status = SSH_GITALY_FEATURES[action] - return unless feature && Gitlab::GitalyClient.feature_enabled?(feature, status: status) + return unless %w[git-receive-pack git-upload-pack].include?(action) + + if action == 'git-receive-pack' + return unless Gitlab::GitalyClient.feature_enabled?( + :ssh_receive_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) + end { repository: repository.gitaly_repository, diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 79b302aae70..063f0d6599c 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -82,6 +82,18 @@ module API end # + # Get a ssh key using the fingerprint + # + get "/authorized_keys" do + fingerprint = params.fetch(:fingerprint) do + Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint + end + key = Key.find_by(fingerprint: fingerprint) + not_found!("Key") if key.nil? + present key, with: Entities::SSHKey + end + + # # Discover user by ssh key or user id # get "/discover" do @@ -91,6 +103,7 @@ module API elsif params[:user_id] user = User.find_by(id: params[:user_id]) end + present user, with: Entities::UserSafe end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b29c5848aef..c99fe3ab5b3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -175,6 +175,7 @@ module API issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute + if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end @@ -277,6 +278,19 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + desc 'List participants for an issue' do + success Entities::UserBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/participants' do + issue = find_project_issue(params[:issue_iid]) + participants = ::Kaminari.paginate_array(issue.participants) + + present paginate(participants), with: Entities::UserBasic, current_user: current_user, project: user_project + end + desc 'Get the user agent details for an issue' do success Entities::UserAgentDetail end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index a116ab3c9bd..9c205514b3a 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -38,6 +38,7 @@ module API builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) + builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project) present paginate(builds), with: Entities::Job end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index e41a1720ac1..81eaf56e48e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,7 +15,7 @@ module API use :pagination end get ':id/labels' do - present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project + present paginate(available_labels_for(user_project)), with: Entities::Label, current_user: current_user, project: user_project end desc 'Create a new label' do @@ -30,7 +30,7 @@ module API post ':id/labels' do authorize! :admin_label, user_project - label = available_labels.find_by(title: params[:name]) + label = available_labels_for(user_project).find_by(title: params[:name]) conflict!('Label already exists') if label priority = params.delete(:priority) diff --git a/lib/api/members.rb b/lib/api/members.rb index 5446f6b54b1..130c6d6da71 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -22,7 +22,7 @@ module API source = find_source(source_type, params[:id]) users = source.users - users = users.merge(User.search(params[:query])) if params[:query] + users = users.merge(User.search(params[:query])) if params[:query].present? present paginate(users), with: Entities::Member, source: source end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 02f2b75ab9d..420aaf1c964 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -24,6 +24,13 @@ module API .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end + def merge_request_pipelines_with_access + authorize! :read_pipeline, user_project + + mr = find_merge_request_with_access(params[:merge_request_iid]) + mr.all_pipelines + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', desc: 'Return opened, closed, merged, or all merge requests' @@ -185,6 +192,16 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end + desc 'Get the participants of a merge request' do + success Entities::UserBasic + end + get ':id/merge_requests/:merge_request_iid/participants' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + participants = ::Kaminari.paginate_array(merge_request.participants) + + present paginate(participants), with: Entities::UserBasic + end + desc 'Get the commits of a merge request' do success Entities::Commit end @@ -204,6 +221,15 @@ module API present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end + desc 'Get the merge request pipelines' do + success Entities::PipelineBasic + end + get ':id/merge_requests/:merge_request_iid/pipelines' do + pipelines = merge_request_pipelines_with_access + + present paginate(pipelines), with: Entities::PipelineBasic + end + desc 'Update a merge request' do success Entities::MergeRequest end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 74b3376a1f3..675c963bae2 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -48,6 +48,7 @@ module API current_user, declared_params(include_missing: false)) .execute(:api, ignore_skip_ci: true, save_on_errors: false) + if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 0cb209a02d0..306dc0e63d7 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -60,6 +60,15 @@ module API update_milestone_for(user_project) end + desc 'Remove a project milestone' + delete ":id/milestones/:milestone_id" do + authorize! :admin_milestone, user_project + + user_project.milestones.find(params[:milestone_id]).destroy + + status(204) + end + desc 'Get all issues for a single project milestone' do success Entities::IssueBasic end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 2ccda1c1aa1..39c03c40bab 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -13,6 +13,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end @@ -142,7 +143,7 @@ module API get ":id/snippets/:snippet_id/user_agent_detail" do authenticated_as_admin! - snippet = Snippet.find_by!(id: params[:id]) + snippet = Snippet.find_by!(id: params[:snippet_id], project_id: params[:id]) return not_found!('UserAgentDetail') unless snippet.user_agent_detail diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fa222bf2b1c..8b5e4f8edcc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -76,9 +76,9 @@ module API def present_projects(projects, options = {}) projects = reorder_projects(projects) - projects = projects.with_statistics if params[:statistics] - projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + projects = projects.with_statistics if params[:statistics] projects = paginate(projects) if current_user @@ -154,6 +154,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 614822509f0..c15c487deb4 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -2,7 +2,7 @@ module API class ProtectedBranches < Grape::API include PaginationParams - BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) before { authorize_admin_project } diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4f36bbd760f..9638c53a1df 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -15,6 +15,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/services.rb b/lib/api/services.rb index a7f44e2869c..51e33e2c686 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -785,7 +785,7 @@ module API service_params = declared_params(include_missing: false).merge(active: true) if service.update_attributes(service_params) - present service, with: Entities::ProjectService, include_passwords: current_user.admin? + present service, with: Entities::ProjectService else render_api_error!('400 Bad Request', 400) end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 6b6a03e3300..c7a460df46a 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -26,6 +26,7 @@ module API optional :token, type: String, desc: 'The token used to validate payloads' optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on tag push events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" end post do diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index fa0bef39602..ac76fece931 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -36,6 +36,7 @@ module API builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) + builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project) present paginate(builds), with: ::API::V3::Entities::Build end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 0ef26aa696a..4f6ea8f502e 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -71,13 +71,14 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end get ":id/repository/commits/:sha", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit - present commit, with: ::API::Entities::CommitDetail + present commit, with: ::API::Entities::CommitDetail, stats: params[:stats] end desc 'Get the diff for a specific commit of a project' do diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb index b90e2061da3..47e54ca85a5 100644 --- a/lib/api/v3/deploy_keys.rb +++ b/lib/api/v3/deploy_keys.rb @@ -3,6 +3,16 @@ module API class DeployKeys < Grape::API before { authenticate! } + helpers do + def add_deploy_keys_project(project, attrs = {}) + project.deploy_keys_projects.create(attrs) + end + + def find_by_deploy_key(project, key_id) + project.deploy_keys_projects.find_by!(deploy_key: key_id) + end + end + get "deploy_keys" do authenticated_as_admin! @@ -18,25 +28,28 @@ module API %w(keys deploy_keys).each do |path| desc "Get a specific project's deploy keys" do - success ::API::Entities::SSHKey + success ::API::Entities::DeployKeysProject end get ":id/#{path}" do - present user_project.deploy_keys, with: ::API::Entities::SSHKey + keys = user_project.deploy_keys_projects.preload(:deploy_key) + + present keys, with: ::API::Entities::DeployKeysProject end desc 'Get single deploy key' do - success ::API::Entities::SSHKey + success ::API::Entities::DeployKeysProject end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end get ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - present key, with: ::API::Entities::SSHKey + key = find_by_deploy_key(user_project, params[:key_id]) + + present key, with: ::API::Entities::DeployKeysProject end desc 'Add new deploy key to currently authenticated user' do - success ::API::Entities::SSHKey + success ::API::Entities::DeployKeysProject end params do requires :key, type: String, desc: 'The new deploy key' @@ -47,24 +60,31 @@ module API params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys.find_by(key: params[:key]) + key = user_project.deploy_keys_projects + .joins(:deploy_key) + .find_by(keys: { key: params[:key] }) + if key - present key, with: ::API::Entities::SSHKey + present key, with: ::API::Entities::DeployKeysProject break end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key - user_project.deploy_keys << key - present key, with: ::API::Entities::SSHKey + added_key = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) + + present added_key, with: ::API::Entities::DeployKeysProject break end # Create a new deploy key - key = DeployKey.new(declared_params(include_missing: false)) - if key.valid? && user_project.deploy_keys << key - present key, with: ::API::Entities::SSHKey + key_attributes = { can_push: !!params[:can_push], + deploy_key_attributes: declared_params.except(:can_push) } + key = add_deploy_keys_project(user_project, key_attributes) + + if key.valid? + present key, with: ::API::Entities::DeployKeysProject else render_validation_error!(key) end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c17b6f45ed8..2ccbb9da1c5 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -207,7 +207,7 @@ module API end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :token, :created_at, :updated_at, :last_used expose :owner, using: ::API::Entities::UserBasic end @@ -257,10 +257,7 @@ module API expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| - field_names = service.fields - .select { |field| options[:include_passwords] || field[:type] != 'password' } - .map { |field| field[:name] } - service.properties.slice(*field_names) + service.properties.slice(*service.api_field_names) end end diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb index bd5eb2175e8..4157462ec2a 100644 --- a/lib/api/v3/labels.rb +++ b/lib/api/v3/labels.rb @@ -11,7 +11,7 @@ module API success ::API::Entities::Label end get ':id/labels' do - present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project + present available_labels_for(user_project), with: ::API::Entities::Label, current_user: current_user, project: user_project end desc 'Delete an existing label' do diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index 684860b553e..46145cac7a5 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -23,7 +23,7 @@ module API source = find_source(source_type, params[:id]) users = source.users - users = users.merge(User.search(params[:query])) if params[:query] + users = users.merge(User.search(params[:query])) if params[:query].present? present paginate(users), with: ::API::Entities::Member, source: source end @@ -67,6 +67,7 @@ module API unless member member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end + if member.persisted? && member.valid? present member.user, with: ::API::Entities::Member, member: member else diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 1d6d823f32b..0a24fea52a3 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -126,6 +126,7 @@ module API if status == :deprecated detail DEPRECATION_MESSAGE end + success ::API::V3::Entities::MergeRequest end get path do diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index c41fee32610..6ba425ba8c7 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 7c260b8d910..a7f0813bf74 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -41,6 +41,7 @@ module API # private or internal, use the more conservative option, private. attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE end + attrs end @@ -174,7 +175,7 @@ module API end get "/search/:query", requirements: { query: /[^\/]+/ } do search_service = Search::GlobalService.new(current_user, search: params[:query]).execute - projects = search_service.objects('projects', params[:page]) + projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) present paginate(projects), with: ::API::V3::Entities::Project @@ -201,6 +202,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index f9a47101e27..5b54734bb45 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end end diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 44ed94d2869..20ca1021c71 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -622,7 +622,7 @@ module API end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService, include_passwords: current_user.admin? + present service, with: Entities::ProjectService end end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 126ec72248e..85613c8ed84 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -97,6 +97,7 @@ module API attrs = declared_params(include_missing: false) UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? present snippet, with: ::API::Entities::PersonalSnippet else diff --git a/lib/backup/database.rb b/lib/backup/database.rb index d97e5d98229..5e6828de597 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -31,6 +31,7 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end compress_wr.close diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 05aa79dc160..f27ce4d2b2b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -108,7 +108,10 @@ module Backup $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 elsif backup_file_list.many? && ENV["BACKUP"].nil? - $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'Found more than one backup:' + # print list of available backups + $progress.puts " " + available_timestamps.join("\n ") + $progress.puts 'Please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end @@ -169,6 +172,10 @@ module Backup @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") end + def available_timestamps + @backup_file_list.map {|item| item.gsub("#{FILE_NAME_SUFFIX}", "")} + end + def connect_to_remote_directory(connection_settings) # our settings use string keys, but Fog expects symbols connection = ::Fog::Storage.new(connection_settings.symbolize_keys) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 2a04c03919d..6715159a1aa 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -47,6 +47,7 @@ module Backup if File.exist?(path_to_wiki_repo) progress.print " * #{display_repo_path(wiki)} ... " + if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index b545b947a2c..65c131e08d9 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -2,16 +2,7 @@ module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter def call - doc.css('pre[lang="mermaid"]').add_class('mermaid') - doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') - - # The `<code></code>` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb - # We want to keep context and consistency, so we the blocks are added for all filters. - # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 - doc.css('pre[lang="mermaid"]').each do |pre| - document = pre.at('code') - document.replace(document.content) - end + doc.css('pre[lang="mermaid"] > code').add_class('js-render-mermaid') doc end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5c197afd782..9bdedeb6615 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -50,15 +50,22 @@ module Banzai end def process_link_to_upload_attr(html_attr) - uri_parts = [html_attr.value] + path_parts = [Addressable::URI.unescape(html_attr.value)] if group - uri_parts.unshift(relative_url_root, 'groups', group.full_path, '-') + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') elsif project - uri_parts.unshift(relative_url_root, project.full_path) + path_parts.unshift(relative_url_root, project.full_path) end - html_attr.value = File.join(*uri_parts) + path = Addressable::URI.escape(File.join(*path_parts)) + + html_attr.value = + if context[:only_path] + path + else + Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s + end end def process_link_to_repository_attr(html_attr) diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index e7a1ec8457d..072d24e5a11 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -9,6 +9,10 @@ module Banzai end def apply_rules + # Special case: relative URLs beginning with `/uploads/` refer to + # user-uploaded files and will be handled elsewhere. + return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/') + apply_file_link_rules! apply_hierarchical_link_rules! apply_relative_link_rules! diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb new file mode 100644 index 00000000000..dae03a179e4 --- /dev/null +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true +module Gitlab + module Auth + class BlockedUserTracker + ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters' + + def self.log_if_user_blocked(env) + message = env.dig('warden.options', :message) + + # Devise calls User#active_for_authentication? on the User model and then + # throws an exception to Warden with User#inactive_message: + # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8 + # + # Since Warden doesn't pass the user record to the failure handler, we + # need to do a database lookup with the username. We can limit the + # lookups to happen when the user was blocked by checking the inactive + # message passed along by Warden. + return unless message == User::BLOCKED_MESSAGE + + login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') + + return unless login.present? + + user = User.by_login(login) + + return unless user&.blocked? + + Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}") + SystemHooksService.new.execute_hooks_for(user, :failed_login) + + true + rescue TypeError + end + end + end +end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index b4114a3ac96..cf02030c577 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -96,9 +96,7 @@ module Gitlab end def ensure_action_dispatch_request(request) - return request if request.is_a?(ActionDispatch::Request) - - ActionDispatch::Request.new(request.env) + ActionDispatch::Request.new(request.env.dup) end def current_request diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb new file mode 100644 index 00000000000..7bffffec94d --- /dev/null +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +# rubocop:disable Metrics/LineLength + +module Gitlab + module BackgroundMigration + class AddMergeRequestDiffCommitsCount + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + def perform(start_id, stop_id) + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + + update = ' + commits_count = ( + SELECT count(*) + FROM merge_request_diff_commits + WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id + )'.squish + + MergeRequestDiff.where(id: start_id..stop_id).update_all(update) + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb index a2cb215c230..ef70f37d5eb 100644 --- a/lib/gitlab/background_migration/copy_column.rb +++ b/lib/gitlab/background_migration/copy_column.rb @@ -28,6 +28,8 @@ module Gitlab UPDATE #{quoted_table} SET #{quoted_copy_to} = #{quoted_copy_from} WHERE id BETWEEN #{start_id} AND #{end_id} + AND #{quoted_copy_from} IS NOT NULL + AND #{quoted_copy_to} IS NULL SQL end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 476c46341ae..4e0121ca34d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -7,6 +7,7 @@ module Gitlab class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength # For bulk_queue_background_migration_jobs_by_range include Database::MigrationHelpers + include ::Gitlab::Utils::StrongMemoize FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze @@ -142,7 +143,9 @@ module Gitlab end def postgresql? - @postgresql ||= Gitlab::Database.postgresql? + strong_memoize(:postgresql) do + Gitlab::Database.postgresql? + end end def can_bulk_insert_and_ignore_duplicates? @@ -150,8 +153,9 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && - Gitlab::Database.version.to_f < 9.5 + strong_memoize(:postgresql_pre_9_5) do + postgresql? && Gitlab::Database.version.to_f < 9.5 + end end def schedule_populate_untracked_uploads_jobs diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 709a901aa77..884a3de8f62 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -63,6 +63,7 @@ module Gitlab log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config + project.repository.create_hooks ProjectCacheWorker.perform_async(project.id) else diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index 85b79362196..c0c666dfb7b 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,6 +1,8 @@ module Gitlab module BareRepositoryImport class Repository + include ::Gitlab::Utils::StrongMemoize + attr_reader :group_path, :project_name, :repo_path def initialize(root_path, repo_path) @@ -41,11 +43,15 @@ module Gitlab private def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') + strong_memoize(:wiki) do + repo_path.end_with?('.wiki.git') + end end def hashed? - @hashed ||= repo_relative_path.include?('@hashed') + strong_memoize(:hashed) do + repo_relative_path.include?('@hashed') + end end def repo_relative_path diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index ef92fc5a0a0..945d70e7a24 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -16,7 +16,7 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :protocol + attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name def initialize( change, user_access:, project:, skip_authorization: false, @@ -51,9 +51,9 @@ module Gitlab end def branch_checks - return unless @branch_name + return unless branch_name - if deletion? && @branch_name == project.default_branch + if deletion? && branch_name == project.default_branch raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] end @@ -61,7 +61,7 @@ module Gitlab end def protected_branch_checks - return unless ProtectedBranch.protected?(project, @branch_name) + return unless ProtectedBranch.protected?(project, branch_name) if forced_push? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] @@ -75,29 +75,29 @@ module Gitlab end def protected_branch_deletion_checks - unless user_access.can_delete_branch?(@branch_name) + unless user_access.can_delete_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] end - unless protocol == 'web' + unless updated_from_web? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end def protected_branch_push_checks if matching_merge_request? - unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] end else - unless user_access.can_push_to_branch?(@branch_name) + unless user_access.can_push_to_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] end end end def tag_checks - return unless @tag_name + return unless tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] @@ -107,40 +107,44 @@ module Gitlab end def protected_tag_checks - return unless ProtectedTag.protected?(project, @tag_name) + return unless ProtectedTag.protected?(project, tag_name) raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? - unless user_access.can_create_tag?(@tag_name) + unless user_access.can_create_tag?(tag_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] end end private + def updated_from_web? + protocol == 'web' + end + def tag_exists? - project.repository.tag_exists?(@tag_name) + project.repository.tag_exists?(tag_name) end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) end def update? - !Gitlab::Git.blank_ref?(@oldrev) && !deletion? + !Gitlab::Git.blank_ref?(oldrev) && !deletion? end def deletion? - Gitlab::Git.blank_ref?(@newrev) + Gitlab::Git.blank_ref?(newrev) end def matching_merge_request? - Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end def lfs_objects_exist_check - lfs_check = Checks::LfsIntegrity.new(project, @newrev) + lfs_check = Checks::LfsIntegrity.new(project, newrev) if lfs_check.objects_missing? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e25916528f4..35eadf6fa93 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -148,6 +148,7 @@ module Gitlab stream.seek(@offset) append = @offset > 0 end + start_offset = @offset open_new_tag @@ -155,6 +156,7 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) handle_section(s) elsif s.scan(/\e([@-_])(.*?)([@-~])/) @@ -168,6 +170,7 @@ module Gitlab else @out << s.scan(/./m) end + @offset += s.matched_size end end @@ -236,8 +239,10 @@ module Gitlab if @style_mask & STYLE_SWITCHES[:bold] != 0 fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end + css_classes << fg_color end + css_classes << @bg_color unless @bg_color.nil? STYLE_SWITCHES.each do |css_class, flag| diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index eb606b57667..55658900628 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -64,10 +64,24 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) - unless validate_string(value) + if validate_string(value) + validate_path(record, attribute, value) + else record.errors.add(attribute, 'should be a string or symbol') end end + + private + + def validate_path(record, attribute, value) + path = CGI.unescape(value.to_s) + + if path.include?('/') + record.errors.add(attribute, 'cannot contain the "/" character') + elsif path == '.' || path == '..' + record.errors.add(attribute, 'cannot be "." or ".."') + end + end end class RegexpValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 9a72de87bab..32cbb7ca6af 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -3,6 +3,8 @@ module Gitlab module Pipeline module Chain class Skip < Chain::Base + include ::Gitlab::Utils::StrongMemoize + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i def perform! @@ -24,7 +26,9 @@ module Gitlab def commit_message_skips_ci? return false unless @pipeline.git_commit_message - @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + strong_memoize(:commit_message_skips_ci) do + !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end end end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index bc97aa63b02..f33c87f554d 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -2,6 +2,8 @@ module Gitlab module Ci module Stage class Seed + include ::Gitlab::Utils::StrongMemoize + attr_reader :pipeline delegate :project, to: :pipeline @@ -50,7 +52,9 @@ module Gitlab private def protected_ref? - @protected_ref ||= project.protected_for?(pipeline.ref) + strong_memoize(:protected_ref) do + project.protected_for?(pipeline.ref) + end end end end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb index 45fd0d4aa07..6c9125647ad 100644 --- a/lib/gitlab/ci/status/build/action.rb +++ b/lib/gitlab/ci/status/build/action.rb @@ -2,6 +2,9 @@ module Gitlab module Ci module Status module Build + ## + # Extended status for playable manual actions. + # class Action < Status::Extended def label if has_action? @@ -12,7 +15,7 @@ module Gitlab end def self.matches?(build, user) - build.action? + build.playable? end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index dcbdf9a64b0..8b3bc3e440d 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -15,7 +15,6 @@ module Gitlab query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_table[:deleted_at].eq(nil)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 9f76967fc77..d32837f5793 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -12,30 +12,40 @@ module Gitlab # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - priv = - if Database.postgresql? - where(privilege_type: 'TRIGGER', table_name: table) - .where('grantee = user') - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), + if Database.postgresql? + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + begin + from(nil) + .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')") + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false + end + else + queries = [ + Grant.select(1) + .from('information_schema.user_privileges') + .where("PRIVILEGE_TYPE = 'SUPER'") + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - union = SQL::Union.new(queries).to_sql + Grant.select(1) + .from('information_schema.schema_privileges') + .where("PRIVILEGE_TYPE = 'TRIGGER'") + .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") + ] - Grant.from("(#{union}) privs") - end + union = SQL::Union.new(queries).to_sql - priv.any? + Grant.from("(#{union}) privs").any? + end end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 33171f83692..dbe6259fce7 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -512,6 +512,7 @@ module Gitlab batch_size: 10_000, interval: 10.minutes ) + unless relation.model < EachBatch raise TypeError, 'The relation must include the EachBatch module' end @@ -524,8 +525,9 @@ module Gitlab install_rename_triggers(table, column, temp_column) # Schedule the jobs that will copy the data from the old column to the - # new one. - relation.each_batch(of: batch_size) do |batch, index| + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + relation.where.not(column => nil).each_batch(of: batch_size) do |batch, index| start_id, end_id = batch.pluck('MIN(id), MAX(id)').first max_index = index @@ -842,6 +844,12 @@ into similar problems in the future (e.g. when new tables are created). def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL + delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + end + model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck('MIN(id), MAX(id)').first diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index d32616862f0..979225dd216 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -26,6 +26,7 @@ module Gitlab move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b669ee5b799..0f897e6316c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,6 +14,7 @@ module Gitlab else @diff_lines = diff_lines end + @raw_lines = @diff_lines.map(&:text) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 37face8e7d0..d3b49b1ec75 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -156,12 +156,14 @@ module Gitlab %W[git apply --3way #{patch_path}] ) do |output, status| puts output + unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') file = line.sub(/\Aerror: patch failed: /, '') memo << file unless file =~ IGNORED_FILES_REGEX end + memo end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e2f7c1d0257..3436306e122 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -10,6 +10,7 @@ module Gitlab def initialize(mail, mail_key) super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) @project_path, @incoming_email_token = m.captures end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 6b53eb4533d..c0edcabc6fd 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,14 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to?(:force_encoding) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - - if message.respond_to?(:frozen?) && message.frozen? - message = message.dup - end - - message.force_encoding("UTF-8") + message = force_encode_utf8(message) return message if message.valid_encoding? # return message if message type is binary @@ -35,6 +28,8 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) + rescue ArgumentError + return nil rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" @@ -54,8 +49,8 @@ module Gitlab end def encode_utf8(message) - return nil unless message.is_a?(String) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + message = force_encode_utf8(message) + return message if message.valid_encoding? detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -69,6 +64,8 @@ module Gitlab else clean(message) end + rescue ArgumentError + return nil end def encode_binary(s) @@ -83,6 +80,15 @@ module Gitlab private + def force_encode_utf8(message) + raise ArgumentError unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + + message = message.dup if message.respond_to?(:frozen?) && message.frozen? + + message.force_encoding("UTF-8") + end + def clean(message) message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") .encode("UTF-8") diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3f7b42456af..dbb8f317afe 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -71,5 +71,16 @@ module Gitlab redis.exists(@redis_shared_state_key) end end + + # Returns the TTL of the Redis key. + # + # This method will return `nil` if no TTL could be obtained. + def ttl + Gitlab::Redis::SharedState.with do |redis| + ttl = redis.ttl(@redis_shared_state_key) + + ttl if ttl.positive? + end + end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 5e426b13ade..8953bc8c148 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -112,6 +112,7 @@ module Gitlab [bug['sCategory'], bug['sPriority']].each do |label| unless label.blank? labels << label + unless @known_labels.include?(label) create_label(label) @known_labels << label @@ -265,6 +266,7 @@ module Gitlab if content.blank? content = '*(No description has been entered for this issue)*' end + body << content body.join("\n\n") @@ -278,6 +280,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f7c35cafaa..71647099f83 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..81e46028752 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -34,7 +34,7 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) end @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -61,11 +70,19 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force + return [] if blob_ids.empty? + + repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| + if is_enabled + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids) + else + blob_ids.lazy + .select { |sha| possible_lfs_blob?(repository, sha) } + .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } + .select(&:lfs_pointer?) + .force + end + end end def binary?(data) @@ -122,13 +139,25 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( @@ -154,8 +183,10 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) - commit = repository.lookup(sha) - root_tree = commit.tree + return unless path + + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree blob_entry = find_entry_by_path(repository, root_tree.oid, path) @@ -235,7 +266,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 145721dea76..768617e2cae 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -15,8 +15,6 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :rugged_commit - def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -241,6 +239,24 @@ module Gitlab end end end + + def extract_signature(repository, commit_id) + repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.extract_signature(commit_id) + else + rugged_extract_signature(repository, commit_id) + end + end + end + + def rugged_extract_signature(repository, commit_id) + begin + Rugged::Commit.extract_signature(repository.rugged, commit_id) + rescue Rugged::OdbError + nil + end + end end def initialize(repository, raw_commit, head = nil) @@ -438,6 +454,16 @@ module Gitlab parent_ids.size > 1 end + def tree_entry(path) + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| + if is_migrated + gitaly_tree_entry(path) + else + rugged_tree_entry(path) + end + end + end + def to_gitaly_commit return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) @@ -498,6 +524,28 @@ module Gitlab SERIALIZE_KEYS end + def gitaly_tree_entry(path) + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry + end + + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + rescue Rugged::TreeError + nil + end + def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 74c9874d590..07b7e811a34 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -15,7 +15,7 @@ module Gitlab @conflicts ||= begin @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a else rugged_list_conflict_files end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index cba638c06db..e5a747cb987 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -41,62 +41,16 @@ module Gitlab io.read end - def rm_project - logger.info "Removing repository <#{repository_absolute_path}>." - FileUtils.rm_rf(repository_absolute_path) - end - - # Move repository from one directory to another - # - # Example: gitlab/gitlab-ci.git -> randx/six.git - # - # Won't work if target namespace directory does not exist - # - def mv_project(new_path) - new_absolute_path = File.join(shard_path, new_path) - - # verify that the source repo exists - unless File.exist?(repository_absolute_path) - logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist." - return false - end - - # ...and that the target repo does not exist - if File.exist?(new_absolute_path) - logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists." - return false - end - - logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>." - FileUtils.mv(repository_absolute_path, new_absolute_path) - end - # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false + Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| + if is_enabled + gitaly_import_repository(source) + else + git_import_repository(source, timeout) + end end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true end def fork_repository(new_shard_path, new_repository_relative_path) @@ -261,6 +215,42 @@ module Gitlab raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") end + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def gitaly_import_repository(source) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output << e.message + false + end + def git_fork_repository(new_shard_path, new_repository_relative_path) from_path = repository_absolute_path to_path = File.join(new_shard_path, new_repository_relative_path) diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index db532600d1b..d94082a3e30 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -10,6 +10,7 @@ module Gitlab DEFAULT_MODE = 0o100644 ACTIONS = %w(create create_dir update move delete).freeze + ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze attr_reader :repository, :raw_index @@ -20,6 +21,11 @@ module Gitlab delegate :read_tree, :get, to: :raw_index + def apply(action, options) + validate_action!(action) + public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend + end + def write_tree raw_index.write_tree(repository.rugged) end @@ -140,6 +146,12 @@ module Gitlab rescue Rugged::IndexError => e raise IndexError, e.message end + + def validate_action!(action) + unless ACTIONS.include?(action.to_s) + raise ArgumentError, "Unknown action '#{action}'" + end + end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..3fb0e2eed93 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 372ce005b94..a3ba9475ad0 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -33,9 +33,9 @@ module Gitlab object end - def initialize(repository, name, target, derefenced_target) + def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) - @dereferenced_target = derefenced_target + @dereferenced_target = dereferenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index 38e9d2a8554..ebe46722890 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -6,7 +6,23 @@ module Gitlab @ref_name = ref_name end - def update(only_branches_matching: [], only_tags_matching: []) + def update(only_branches_matching: []) + @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| + if is_enabled + gitaly_update(only_branches_matching) + else + rugged_update(only_branches_matching) + end + end + end + + private + + def gitaly_update(only_branches_matching) + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) + end + + def rugged_update(only_branches_matching) local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) @@ -15,8 +31,8 @@ module Gitlab delete_refs(local_branches, remote_branches) - local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) - remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + local_tags = refs_obj(@repository.tags) + remote_tags = refs_obj(@repository.remote_tags(@ref_name)) updated_tags = changed_refs(local_tags, remote_tags) @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? @@ -24,8 +40,6 @@ module Gitlab delete_refs(local_tags, remote_tags) end - private - def refs_obj(refs, only_refs_matching: []) refs.each_with_object({}) do |ref, refs| next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aec85f971ca..d7c712e75c5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -490,19 +490,17 @@ module Gitlab return [] end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end + log_by_shell(sha, options) end def count_commits(options) + count_commits_options = process_count_commits_options(options) + gitaly_migrate(:count_commits) do |is_enabled| if is_enabled - count_commits_by_gitaly(options) + count_commits_by_gitaly(count_commits_options) else - count_commits_by_shelling_out(options) + count_commits_by_shelling_out(count_commits_options) end end end @@ -540,8 +538,8 @@ module Gitlab end # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to) - count_commits(ref: "#{from}..#{to}") + def count_commits_between(from, to, options = {}) + count_commits(from: from, to: to, **options) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -569,7 +567,21 @@ module Gitlab end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -605,37 +617,6 @@ module Gitlab end end - # Returns branch names collection that contains the special commit(SHA1 - # or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branch_names_contains(commit) - branches_contains(commit).map { |c| c.name } - end - - # Returns branch collection that contains the special commit(SHA1 or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branches_contains(commit) - commit_obj = rugged.rev_parse(commit) - parent = commit_obj.parents.first unless commit_obj.parents.empty? - - walker = Rugged::Walker.new(rugged) - - rugged.branches.select do |branch| - walker.push(branch.target_id) - walker.hide(parent) if parent - result = walker.any? { |c| c.oid == commit_obj.oid } - walker.reset - - result - end - end - # Get refs hash which key is SHA1 # and value is a Rugged::Reference def refs_hash @@ -652,6 +633,7 @@ module Gitlab end end end + @refs_hash end @@ -1101,12 +1083,12 @@ module Gitlab end end - def write_ref(ref_path, ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - - input = "update #{ref_path}\x00#{ref}\x00\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + def write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1161,23 +1143,13 @@ module Gitlab end def fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - - # Notice that this feature flag is not for `fetch_repository_as_mirror` - # as a whole but for the fetching mechanism (file path or gitaly-ssh). - url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| if is_enabled - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - [GITALY_INTERNAL_URL, repository.fetch_env] + gitaly_remote_client.fetch_internal_remote(repository) else - [repository.path, nil] + rugged_fetch_repository_as_mirror(repository) end end - - add_remote(remote_name, url, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: env) - ensure - remove_remote(remote_name) end def blob_at(sha, path) @@ -1185,7 +1157,7 @@ module Gitlab end # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: nil) + def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end @@ -1216,26 +1188,31 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha + gitaly_migrate(:rebase) do |is_enabled| + if is_enabled + gitaly_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + else + git_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end end end def rebase_in_progress?(rebase_id) - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + gitaly_migrate(:rebase_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.rebase_in_progress?(rebase_id) + else + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) @@ -1291,6 +1268,41 @@ module Gitlab success || gitlab_projects_error end + def bundle_to_disk(save_path) + gitaly_migrate(:bundle_to_disk) do |is_enabled| + if is_enabled + gitaly_repository_client.create_bundle(save_path) + else + run_git!(%W(bundle create #{save_path} --all)) + end + end + + true + end + + # rubocop:disable Metrics/ParameterLists + def multi_action( + user, branch_name:, message:, actions:, + author_email: nil, author_name: nil, + start_branch_name: nil, start_repository: self) + + gitaly_migrate(:operation_user_commit_files) do |is_enabled| + if is_enabled + gitaly_operation_client.user_commit_files(user, branch_name, + message, actions, author_email, author_name, + start_branch_name, start_repository) + else + rugged_multi_action(user, branch_name, message, actions, + author_email, author_name, start_branch_name, start_repository) + end + end + end + # rubocop:enable Metrics/ParameterLists + + def write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path if full_path.present? + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1319,6 +1331,10 @@ module Gitlab @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) end + def gitaly_blob_client + @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self) + end + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) end @@ -1335,6 +1351,25 @@ module Gitlab private + def shell_write_ref(ref_path, ref, old_ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") + + input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + end + + def rugged_write_ref(ref_path, ref) + rugged.references.create(ref_path, ref, force: true) + rescue Rugged::ReferenceError => ex + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + rescue Rugged::OSError => ex + raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + end + def fresh_worktree?(path) File.exist?(path) && !clean_stuck_worktree(path) end @@ -1440,14 +1475,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1461,27 +1489,34 @@ module Gitlab end end - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) end - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a + def process_count_commits_options(options) + if options[:from] || options[:to] + ref = + if options[:left_right] # Compare with merge-base for left-right + "#{options[:from]}...#{options[:to]}" + else + "#{options[:from]}..#{options[:to]}" + end + + options.merge(ref: ref) + + elsif options[:ref] && options[:left_right] + from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] + + options.merge(from: from, to: to) + else + options + end end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the - # complexity is such that we should migrate it as Ruby before trying to - # do it in Go. def log_by_shell(sha, options) limit = options[:limit].to_i offset = options[:offset].to_i @@ -1683,20 +1718,59 @@ module Gitlab end def count_commits_by_gitaly(options) - gitaly_commit_client.commit_count(options[:ref], options) + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end end def count_commits_by_shelling_out(options) + cmd = count_commits_shelling_command(options) + + raw_output = IO.popen(cmd) { |io| io.read } + + process_count_commits_raw_output(raw_output, options) + end + + def count_commits_shelling_command(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] cmd << "--max-count=#{options[:max_count]}" if options[:max_count] + cmd << "--left-right" if options[:left_right] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? + cmd + end - raw_output = IO.popen(cmd) { |io| io.read } + def process_count_commits_raw_output(raw_output, options) + if options[:left_right] + result = raw_output.scan(/\d+/).map(&:to_i) - raw_output.to_i + if result.sum != options[:max_count] + result + else # Reaching max count, right is not accurate + right_option = + process_count_commits_options(options + .except(:left_right, :from, :to) + .merge(ref: options[:to])) + + right = count_commits_by_shelling_out(right_option) + + [result.first, right] # left should be accurate in the first call + end + else + raw_output.to_i + end end def gitaly_ls_files(ref) @@ -1916,6 +1990,40 @@ module Gitlab tree_id end + def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end + + def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repo_path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1966,6 +2074,49 @@ module Gitlab false end + def rugged_fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + + def rugged_multi_action( + user, branch_name, message, actions, author_email, author_name, + start_branch_name, start_repository) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 4974205b8fd..f8b2e7e0e21 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -95,7 +95,7 @@ module Gitlab object_output.map do |output_line| sha, path = output_line.split(' ', 2) - next if require_path && path.blank? + next if require_path && path.to_s.empty? sha end.reject(&:nil?) diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 1307f400700..0a4e557b59b 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index a06bac4414f..669ae11a423 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -1,7 +1,7 @@ module Gitlab module Git class WikiPage - attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data # This class is meant to be serializable so that it can be constructed # by Gitaly and sent over the network to GitLab. @@ -21,6 +21,7 @@ module Gitlab @raw_data = gollum_page.raw_data @name = gollum_page.name @historical = gollum_page.historical? + @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) @version = version end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 4507ea923b4..6bd256f57c7 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -1,6 +1,8 @@ require 'base64' require 'gitaly' +require 'grpc/health/v1/health_pb' +require 'grpc/health/v1/health_services_pb' module Gitlab module GitalyClient @@ -69,14 +71,27 @@ module Gitlab @stubs ||= {} @stubs[storage] ||= {} @stubs[storage][name] ||= begin - klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) - addr = address(storage) - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + klass = stub_class(name) + addr = stub_address(storage) klass.new(addr, :this_channel_is_insecure) end end end + def self.stub_class(name) + if name == :health_check + Grpc::Health::V1::Health::Stub + else + Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + end + end + + def self.stub_address(storage) + addr = address(storage) + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + addr + end + def self.clear_stubs! MUTEX.synchronize do @stubs = nil diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index a250eb75bd4..ee36684197b 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -32,6 +32,26 @@ module Gitlab binary: Gitlab::Git::Blob.binary?(data) ) end + + def batch_lfs_pointers(blob_ids) + request = Gitaly::GetLFSPointersRequest.new( + repository: @gitaly_repo, + blob_ids: blob_ids + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) + + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fed05bb6c64..33a8d3e5612 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -125,11 +125,11 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: ref + revision: encode_binary(ref) ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? - request.path = options[:path] if options[:path].present? + request.path = encode_binary(options[:path]) if options[:path].present? request.max_count = options[:max_count] if options[:max_count].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count @@ -177,7 +177,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) - rescue GRPC::Unknown # If no repository is found, happens mainly during testing + rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] end @@ -282,6 +282,23 @@ module Gitlab end end + def extract_signature(commit_id) + request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) + response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request) + + signature = ''.b + signed_text = ''.b + + response.each do |message| + signature << message.signature + signed_text << message.signed_text + end + + return if signature.blank? && signed_text.blank? + + [signature, signed_text] + end + private def call_commit_diff(request_params, options = {}) diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb new file mode 100644 index 00000000000..97c13d1fdb0 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class ConflictFilesStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_file = nil + + @rpc_response.each do |msg| + msg.files.each do |gitaly_file| + if gitaly_file.header + yield current_file if current_file + + current_file = file_from_gitaly_header(gitaly_file.header) + else + current_file.content << gitaly_file.content + end + end + end + + yield current_file if current_file + end + + private + + def file_from_gitaly_header(header) + Gitlab::Git::Conflict::File.new( + Gitlab::GitalyClient::Util.git_repository(header.repository), + header.commit_oid, + conflict_from_gitaly_file_header(header), + '' + ) + end + + def conflict_from_gitaly_file_header(header) + { + ours: { path: header.our_path, mode: header.our_mode }, + theirs: { path: header.their_path } + } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 40f032cf873..e14734495a8 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -20,7 +20,16 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) - files_from_response(response).to_a + GitalyClient::ConflictFilesStitcher.new(response) + end + + def conflicts? + list_conflict_files.any? + rescue GRPC::FailedPrecondition + # The server raises this exception when it encounters ConflictSideMissing, which + # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent + # file in one of the trees. + true end def resolve_conflicts(target_repository, resolution, source_branch, target_branch) @@ -58,38 +67,6 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly ) end - - def files_from_response(response) - files = [] - - response.each do |msg| - msg.files.each do |gitaly_file| - if gitaly_file.header - files << file_from_gitaly_header(gitaly_file.header) - else - files.last.content << gitaly_file.content - end - end - end - - files - end - - def file_from_gitaly_header(header) - Gitlab::Git::Conflict::File.new( - Gitlab::GitalyClient::Util.git_repository(header.repository), - header.commit_oid, - conflict_from_gitaly_file_header(header), - '' - ) - end - - def conflict_from_gitaly_file_header(header) - { - ours: { path: header.our_path, mode: header.our_mode }, - theirs: { path: header.their_path } - } - end end end end diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb new file mode 100644 index 00000000000..6c1213f5e20 --- /dev/null +++ b/lib/gitlab/gitaly_client/health_check_service.rb @@ -0,0 +1,19 @@ +module Gitlab + module GitalyClient + class HealthCheckService + def initialize(storage) + @storage = storage + end + + # Sends a gRPC health ping to the Gitaly server for the storage shard. + def check + request = Grpc::Health::V1::HealthCheckRequest.new + response = GitalyClient.call(@storage, :health_check, :check, request, timeout: GitalyClient.fast_timeout) + + { success: response&.status == :SERVING } + rescue GRPC::BadStatus => e + { success: false, message: e.to_s } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index ae1753ff0ae..c2b4155e6a5 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -3,6 +3,8 @@ module Gitlab class OperationService include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -52,6 +54,7 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) + if response.pre_receive_error.present? raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) end @@ -146,6 +149,77 @@ module Gitlab start_repository: start_repository) end + def user_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + request = Gitaly::UserRebaseRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + rebase_id: rebase_id.to_s, + branch: encode_binary(branch), + branch_sha: branch_sha, + remote_repository: remote_repository.gitaly_repository, + remote_branch: encode_binary(remote_branch) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_rebase, + request, + remote_storage: remote_repository.storage + ) + + if response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + elsif response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + else + response.rebase_sha + end + end + + def user_commit_files( + user, branch_name, commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + req_enum = Enumerator.new do |y| + header = user_commit_files_request_header(user, branch_name, + commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + y.yield Gitaly::UserCommitFilesRequest.new(header: header) + + actions.each do |action| + action_header = user_commit_files_action_header(action) + y.yield Gitaly::UserCommitFilesRequest.new( + action: Gitaly::UserCommitFilesAction.new(header: action_header) + ) + + reader = binary_stringio(action[:content]) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::UserCommitFilesRequest.new( + action: Gitaly::UserCommitFilesAction.new(content: chunk) + ) + end + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, + :user_commit_files, req_enum, remote_storage: start_repository.storage) + + if (pre_receive_error = response.pre_receive_error.presence) + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + + if (index_error = response.index_error.presence) + raise Gitlab::Git::Index::IndexError, index_error + end + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) @@ -183,6 +257,33 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end end + + def user_commit_files_request_header( + user, branch_name, commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + Gitaly::UserCommitFilesRequestHeader.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + branch_name: encode_binary(branch_name), + commit_message: encode_binary(commit_message), + commit_author_name: encode_binary(author_name), + commit_author_email: encode_binary(author_email), + start_branch_name: encode_binary(start_branch_name), + start_repository: start_repository.gitaly_repository + ) + end + + def user_commit_files_action_header(action) + Gitaly::UserCommitFilesActionHeader.new( + action: action[:action].upcase.to_sym, + file_path: encode_binary(action[:file_path]), + previous_path: encode_binary(action[:previous_path]), + base64_content: action[:encoding] == 'base64' + ) + rescue RangeError + raise ArgumentError, "Unknown action '#{action[:action]}'" + end end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 5bce1009878..f8e2a27f3fe 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -14,12 +14,18 @@ module Gitlab request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) - response.flat_map do |message| - message.branches.map do |branch| - target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) - Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) - end - end + consume_find_all_branches_response(response) + end + + def merged_branches(branch_names = []) + request = Gitaly::FindAllBranchesRequest.new( + repository: @gitaly_repo, + merged_only: true, + merged_branches: branch_names.map { |s| encode_binary(s) } + ) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + consume_find_all_branches_response(response) end def default_branch_name @@ -62,7 +68,7 @@ module Gitlab request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) - consume_branches_response(response) + consume_find_local_branches_response(response) end def tags @@ -151,7 +157,7 @@ module Gitlab enum_value end - def consume_branches_response(response) + def consume_find_local_branches_response(response) response.flat_map do |message| message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @@ -164,6 +170,15 @@ module Gitlab end end + def consume_find_all_branches_response(response) + response.flat_map do |message| + message.branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + def consume_tags_response(response) response.flat_map do |message| message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 9218f6cfd68..58c356edfd1 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -1,16 +1,20 @@ module Gitlab module GitalyClient class RemoteService + MAX_MSG_SIZE = 128.kilobytes.freeze + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage end - def add_remote(name, url, mirror_refmap) + def add_remote(name, url, mirror_refmaps) request = Gitaly::AddRemoteRequest.new( - repository: @gitaly_repo, name: name, url: url, - mirror_refmap: mirror_refmap.to_s + repository: @gitaly_repo, + name: name, + url: url, + mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) ) GitalyClient.call(@storage, :remote_service, :add_remote, request) @@ -23,6 +27,44 @@ module Gitlab response.result end + + def fetch_internal_remote(repository) + request = Gitaly::FetchInternalRemoteRequest.new( + repository: @gitaly_repo, + remote_repository: repository.gitaly_repository + ) + + response = GitalyClient.call(@storage, :remote_service, + :fetch_internal_remote, request, + remote_storage: repository.storage) + + response.result + end + + def update_remote_mirror(ref_name, only_branches_matching) + req_enum = Enumerator.new do |y| + y.yield Gitaly::UpdateRemoteMirrorRequest.new( + repository: @gitaly_repo, + ref_name: ref_name + ) + + current_size = 0 + + slices = only_branches_matching.slice_before do |branch_name| + current_size += branch_name.bytesize + + next false if current_size < MAX_MSG_SIZE + + current_size = 0 + end + + slices.each do |slice| + y.yield Gitaly::UpdateRemoteMirrorRequest.new(only_branches_matching: slice) + end + end + + GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum) + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d43d80da960..654a3c314f1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -43,8 +43,11 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) - request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:) + request = Gitaly::FetchRemoteRequest.new( + repository: @gitaly_repo, remote: remote, force: forced, + no_tags: no_tags, timeout: timeout + ) if ssh_auth&.ssh_import? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? @@ -97,6 +100,38 @@ module Gitlab ) end + def import_repository(source) + request = Gitaly::CreateRepositoryFromURLRequest.new( + repository: @gitaly_repo, + url: source + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_url, + request, + timeout: GitalyClient.default_timeout + ) + end + + def rebase_in_progress?(rebase_id) + request = Gitaly::IsRebaseInProgressRequest.new( + repository: @gitaly_repo, + rebase_id: rebase_id.to_s + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :is_rebase_in_progress, + request, + timeout: GitalyClient.default_timeout + ) + + response.in_progress + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, @@ -126,6 +161,23 @@ module Gitlab return response.error.b, 1 end end + + def create_bundle(save_path) + request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + :create_bundle, + request, + timeout: GitalyClient.default_timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 5da9befa08e..4f160e4a447 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -14,6 +14,8 @@ module Gitlab # puts label.name # end class Client + include ::Gitlab::Utils::StrongMemoize + attr_reader :octokit # A single page of data and the corresponding page number. @@ -173,7 +175,9 @@ module Gitlab end def rate_limiting_enabled? - @rate_limiting_enabled ||= api_endpoint.include?('.github.com') + strong_memoize(:rate_limiting_enabled) do + api_endpoint.include?('.github.com') + end end def api_endpoint diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index ab38c0c3e34..46b49128140 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -302,6 +302,7 @@ module Gitlab else "#{project.namespace.full_path}/#{name}##{id}" end + text = "~~#{text}~~" if deleted text end @@ -329,6 +330,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? @@ -352,6 +354,7 @@ module Gitlab if content.blank? content = "*(No description has been entered for this issue)*" end + body << content if attachments.any? diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 0f4ba6f83fc..672b5579dfd 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -4,12 +4,8 @@ module Gitlab def initialize(commit) @commit = commit - @signature_text, @signed_text = - begin - Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha) - rescue Rugged::OdbError - nil - end + repo = commit.project.repository.raw_repository + @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha) end def has_signature? diff --git a/lib/gitlab/grape_logging/loggers/user_logger.rb b/lib/gitlab/grape_logging/loggers/user_logger.rb new file mode 100644 index 00000000000..fa172861967 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/user_logger.rb @@ -0,0 +1,18 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the user who performed the Grape API action by retrieving +# the user context from the request environment. +module Gitlab + module GrapeLogging + module Loggers + class UserLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + params = request.env[::API::Helpers::API_USER_ENV] + + return {} unless params + + params.slice(:user_id, :username) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb new file mode 100644 index 00000000000..11416c002e3 --- /dev/null +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -0,0 +1,53 @@ +module Gitlab + module HealthChecks + class GitalyCheck + extend BaseAbstractCheck + + METRIC_PREFIX = 'gitaly_health_check'.freeze + + class << self + def readiness + repository_storages.map do |storage_name| + check(storage_name) + end + end + + def metrics + repository_storages.flat_map do |storage_name| + result, elapsed = with_timing { check(storage_name) } + labels = { shard: storage_name } + + [ + metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_latency_seconds", elapsed, **labels) + ].flatten + end + end + + def check(storage_name) + serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) + result = serv.check + HealthChecks::Result.new(result[:success], result[:message], shard: storage_name) + end + + private + + def metric_prefix + METRIC_PREFIX + end + + def successful?(result) + result[:success] + end + + def repository_storages + storages.keys + end + + def storages + Gitlab.config.repositories.storages + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index e29dd0d5b0e..f9b1a3caf5e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,7 +7,6 @@ module Gitlab closed_at confidential created_at - deleted_at description due_date id diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae9b68eb648..aff786864f2 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -5,7 +5,6 @@ module Gitlab assignee_id author_id created_at - deleted_at description head_pipeline_id id diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 2066005dddc..af203ff711d 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.1'.freeze + VERSION = '0.2.2'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 0135b3c6f22..25399f307f2 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -11,8 +11,9 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def git_bundle(repo_path:, bundle_path:) - execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + def git_clone_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path})) + Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) end def mkdir_p(path) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 989342389bc..5c971564a73 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -17,12 +17,16 @@ module Gitlab def import mkdir_p(@shared.export_path) + remove_symlinks! + wait_for_archived_file do decompress_archive end rescue => e @shared.error(e) false + ensure + remove_symlinks! end private @@ -43,7 +47,7 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result - remove_symlinks! + result end def remove_symlinks! diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f2b193c79cb..2daed10f678 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -49,8 +49,8 @@ project_tree: - :author - events: - :push_event_payload - - :stages - - :statuses + - stages: + - :statuses - :auto_devops - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c518943be59..4b5f9f3a926 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -148,6 +148,7 @@ module Gitlab else relation_hash = relation_item[sub_relation.to_s] end + [relation_hash, sub_relation] end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d7d1b05e8b9..cb711a83433 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -62,6 +62,7 @@ module Gitlab when :notes then setup_note when :project_label, :project_labels then setup_label when :milestone, :milestones then setup_milestone + when 'Ci::Pipeline' then setup_pipeline else @relation_hash['project_id'] = @project.id end @@ -112,9 +113,7 @@ module Gitlab @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') - imported_object do |object| - object.commit_id = nil - end + imported_object elsif @relation_name == :merge_requests MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else @@ -182,8 +181,9 @@ module Gitlab end def imported_object - yield(existing_or_new_object) if block_given? - existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end existing_or_new_object rescue ActiveRecord::RecordNotUnique @@ -211,6 +211,14 @@ module Gitlab @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end + def setup_pipeline + @relation_hash.fetch('stages').each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @@ -259,6 +267,7 @@ module Gitlab else %w[title group_id] end + finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 32ca2809b2f..d0e5cfcfd3e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) + git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index a7028a32570..695462c7dd2 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -21,7 +21,7 @@ module Gitlab def bundle_to_disk mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + @project.repository.bundle_to_disk(@full_path) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 6130c124dd1..2daeba90a51 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -37,7 +37,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 9fd0b709ef2..d03cbc880fd 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -9,7 +9,11 @@ module Gitlab end def export_path - @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) + end + + def archive_path + @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path) end def error(error) @@ -21,6 +25,14 @@ module Gitlab private + def relative_path + File.join(opts[:relative_path], SecureRandom.hex) + end + + def relative_archive_path + File.join(opts[:relative_path], '..') + end + def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1e6722a7bba..5fa2e101e29 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -10,7 +10,7 @@ module Gitlab def bundle_to_disk(full_path) mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: full_path) + @wiki.repository.bundle_to_disk(full_path) rescue => e @shared.error(e) false diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb new file mode 100644 index 00000000000..f85b6e9197f --- /dev/null +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -0,0 +1,23 @@ +module Gitlab + # + # Calculates the fingerprint of a given key without using + # openssh key validations. For this reason, only use + # for calculating the fingerprint to find the key with it. + # + # DO NOT use it for checking the validity of a ssh key. + # + class InsecureKeyFingerprint + attr_accessor :key + + # + # Gets the base64 encoded string representing a rsa or dsa key + # + def initialize(key_base64) + @key = key_base64 + end + + def fingerprint + OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') + end + end +end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 8d8c441a4b1..bf6981035f4 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -36,7 +36,11 @@ module Gitlab def complete_command(namespace_name) return unless chart - "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" + if chart_values_file + "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null" + else + "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" + end end def install_dps_command diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 233f6bf6227..a3216759cae 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -10,10 +10,12 @@ module Gitlab def generate spec = { containers: [container_specification], restartPolicy: 'Never' } + if command.chart_values_file - generate_config_map - spec['volumes'] = volumes_specification + create_config_map + spec[:volumes] = volumes_specification end + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end @@ -34,19 +36,39 @@ module Gitlab end def labels - { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } + { + 'gitlab.org/action': 'install', + 'gitlab.org/application': command.name + } end def metadata - { name: command.pod_name, namespace: namespace_name, labels: labels } + { + name: command.pod_name, + namespace: namespace_name, + labels: labels + } end def volume_mounts_specification - [{ name: 'config-volume', mountPath: '/etc/config' }] + [ + { + name: 'configuration-volume', + mountPath: "/data/helm/#{command.name}/config" + } + ] end def volumes_specification - [{ name: 'config-volume', configMap: { name: 'values-config' } }] + [ + { + name: 'configuration-volume', + configMap: { + name: 'values-content-configuration', + items: [{ key: 'values', path: 'values.yaml' }] + } + } + ] end def generate_pod_env(command) @@ -57,10 +79,10 @@ module Gitlab }.map { |key, value| { name: key, value: value } } end - def generate_config_map + def create_config_map resource = ::Kubeclient::Resource.new - resource.metadata = { name: 'values-config', namespace: namespace_name } - resource.data = YAML.load_file(command.chart_values_file) + resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } } + resource.data = { values: File.read(command.chart_values_file) } kubeclient.create_config_map(resource) end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 0afaa2306b5..76863e77dc3 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -74,7 +74,7 @@ module Gitlab def user_options(fields, value, limit) options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + attributes: Gitlab::LDAP::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index c8f19cd52d5..cde60addcf7 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -42,6 +42,7 @@ module Gitlab else self.class.invalid_provider(provider) end + @options = config_for(@provider) # Use @provider, not provider end @@ -148,7 +149,7 @@ module Gitlab def default_attributes { - 'username' => %w(uid userid sAMAccountName), + 'username' => %w(uid sAMAccountName userid), 'email' => %w(mail email userPrincipalName), 'name' => 'cn', 'first_name' => 'givenName', diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 38d7a9ba2f5..e81cec6ba1a 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -6,6 +6,8 @@ module Gitlab # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + InvalidEntryError = Class.new(StandardError) + attr_accessor :entry, :provider def self.find_by_uid(uid, adapter) @@ -29,11 +31,12 @@ module Gitlab def self.ldap_attributes(config) [ - 'dn', # Used in `dn` - config.uid, # Used in `uid` - *config.attributes['name'], # Used in `name` - *config.attributes['email'] # Used in `email` - ] + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq end def self.normalize_dn(dn) @@ -60,6 +63,8 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider + + validate_entry end def name @@ -71,7 +76,13 @@ module Gitlab end def username - uid + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first end def email @@ -104,6 +115,19 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end + + def validate_entry + allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) + + # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. + entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } + invalid_attrs = entry_attrs - allowed_attrs + + if invalid_attrs.any? + raise InvalidEntryError, + "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" + end + end end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 877cebf6786..ef44a13df51 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -169,6 +169,7 @@ module Gitlab end end end + @pool end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index fee741b47be..cc1e92480be 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -47,6 +47,7 @@ module Gitlab else value = decorate_params_value(value, @request.params[key], tmp_path) end + @request.update_param(key, value) end @@ -60,6 +61,7 @@ module Gitlab unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end + path_key, path_value = path_hash.first unless value_hash.is_a?(Hash) && value_hash[path_key] diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index c22d0a84860..43921a8c1c0 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -37,6 +37,7 @@ module Gitlab else per_page - first_collection_last_page_size end + hash[page] = second_collection.page(second_collection_page) .per(per_page - paginated_first_collection(page).size) .padding(offset) diff --git a/lib/gitlab/o_auth.rb b/lib/gitlab/o_auth.rb new file mode 100644 index 00000000000..5ad8d83bd6e --- /dev/null +++ b/lib/gitlab/o_auth.rb @@ -0,0 +1,6 @@ +module Gitlab + module OAuth + SignupDisabledError = Class.new(StandardError) + SigninDisabledForProviderError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index d33f33d192f..fff9360ea27 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -5,8 +5,6 @@ # module Gitlab module OAuth - SignupDisabledError = Class.new(StandardError) - class User attr_accessor :auth_hash, :gl_user @@ -29,7 +27,8 @@ module Gitlab end def save(provider = 'OAuth') - unauthorized_to_create unless gl_user + raise SigninDisabledForProviderError if oauth_provider_disabled? + raise SignupDisabledError unless gl_user block_after_save = needs_blocking? @@ -226,8 +225,10 @@ module Gitlab Gitlab::AppLogger end - def unauthorized_to_create - raise SignupDisabledError + def oauth_provider_disabled? + Gitlab::CurrentSettings.current_application_settings + .disabled_oauth_sign_in_sources + .include?(auth_hash.provider) end end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e73245b82c1..e29e168fc5a 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -6,6 +6,7 @@ module Gitlab EXPIRY_TIME = 5.minutes def self.enabled?(user = nil) + return true if Rails.env.development? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb new file mode 100644 index 00000000000..95d94b3cc68 --- /dev/null +++ b/lib/gitlab/profiler.rb @@ -0,0 +1,142 @@ +# coding: utf-8 +module Gitlab + module Profiler + FILTERED_STRING = '[FILTERED]'.freeze + + IGNORE_BACKTRACES = %w[ + lib/gitlab/i18n.rb + lib/gitlab/request_context.rb + config/initializers + lib/gitlab/database/load_balancing/ + lib/gitlab/etag_caching/ + lib/gitlab/metrics/ + lib/gitlab/middleware/ + lib/gitlab/performance_bar/ + lib/gitlab/request_profiler/ + lib/gitlab/profiler.rb + ].freeze + + # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) + # and returns the ruby-prof profile result. Formatting that result is the + # caller's responsibility. Requests are GET requests unless post_data is + # passed. + # + # Optional arguments: + # - logger: will be used for SQL logging, including a summary at the end of + # the log file of the total time spent per model class. + # + # - post_data: a string of raw POST data to use. Changes the HTTP verb to + # POST. + # + # - user: a user to authenticate as. Only works if the user has a valid + # personal access token. + # + # - private_token: instead of providing a user instance, the token can be + # given as a string. Takes precedence over the user option. + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) + app = ActionDispatch::Integration::Session.new(Rails.application) + verb = :get + headers = {} + + if post_data + verb = :post + headers['Content-Type'] = 'application/json' + end + + if user + private_token ||= user.personal_access_tokens.active.pluck(:token).first + end + + headers['Private-Token'] = private_token if private_token + logger = create_custom_logger(logger, private_token: private_token) + + RequestStore.begin! + + # Make an initial call for an asset path in development mode to avoid + # sprockets dominating the profiler output. + ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? + + # Rails loads internationalization files lazily the first time a + # translation is needed. Running this prevents this overhead from showing + # up in profiles. + ::I18n.t('.')[:test_string] + + # Remove API route mounting from the profile. + app.get('/api/v4/users') + + result = with_custom_logger(logger) do + RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + end + + RequestStore.end! + + log_load_times_by_model(logger) + + result + end + + def self.create_custom_logger(logger, private_token: nil) + return unless logger + + logger.dup.tap do |new_logger| + new_logger.instance_variable_set(:@private_token, private_token) + + class << new_logger + attr_reader :load_times_by_model, :private_token + + def debug(message, *) + message.gsub!(private_token, FILTERED_STRING) if private_token + + _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/) + + if type && time + @load_times_by_model ||= {} + @load_times_by_model[type] ||= 0 + @load_times_by_model[type] += time.to_f + end + + super + + backtrace = Rails.backtrace_cleaner.clean(caller) + + backtrace.each do |caller_line| + next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) + + stripped_caller_line = caller_line.sub("#{Rails.root}/", '') + + super(" ↳ #{stripped_caller_line}") + end + end + end + end + end + + def self.with_custom_logger(logger) + original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging + original_activerecord_logger = ActiveRecord::Base.logger + original_actioncontroller_logger = ActionController::Base.logger + + if logger + ActiveSupport::LogSubscriber.colorize_logging = false + ActiveRecord::Base.logger = logger + ActionController::Base.logger = logger + end + + result = yield + + ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging + ActiveRecord::Base.logger = original_activerecord_logger + ActionController::Base.logger = original_actioncontroller_logger + + result + end + + def self.log_load_times_by_model(logger) + return unless logger.respond_to?(:load_times_by_model) + + logger.load_times_by_model.to_a.sort_by(&:last).reverse.each do |(model, time)| + logger.info("#{model} total: #{time.round(2)}ms") + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index e2662fc362b..4823f703ba4 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -20,7 +20,7 @@ module Gitlab when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else - super + super(scope, page, false) end end @@ -44,25 +44,20 @@ module Gitlab ref = nil filename = nil basename = nil + data = "" startline = 0 - result.each_line.each_with_index do |line, index| - matches = line.match(/^(?<ref>[^:]*):(?<filename>.*):(?<startline>\d+):/) - if matches + result.strip.each_line.each_with_index do |line, index| + prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>.*)\x00(?<startline>\d+)\x00/)&.tap do |matches| ref = matches[:ref] filename = matches[:filename] startline = matches[:startline] startline = startline.to_i - index extname = Regexp.escape(File.extname(filename)) basename = filename.sub(/#{extname}$/, '') - break end - end - - data = "" - result.each_line do |line| - data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') + data << line.sub(prefix.to_s, '') end FoundBlob.new( diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 3ebfa3bd4b8..c0878a34fb1 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -126,6 +126,7 @@ module Gitlab command << match_data[1] unless match_data[1].empty? commands << command end + content = substitution.perform_substitution(self, content) end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 8ad06480575..4178b436acf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -24,6 +24,7 @@ module Gitlab # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] end + size end @@ -104,6 +105,7 @@ module Gitlab db_numbers = queries["db"] if queries.key?("db") config[:db] = db_numbers[0].to_i if db_numbers.any? end + config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2c7b8af83f2..7ab85e1c35c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -37,7 +37,7 @@ module Gitlab end def environment_name_regex_chars - 'a-zA-Z0-9_/\\$\\{\\}\\. -' + 'a-zA-Z0-9_/\\$\\{\\}\\. \\-' end def environment_name_regex @@ -67,7 +67,7 @@ module Gitlab end def build_trace_section_regex - @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index ca48c6df602..7362514167f 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -40,19 +40,21 @@ module Gitlab @default_project_filter = default_project_filter end - def objects(scope, page = nil) - case scope - when 'projects' - projects.page(page).per(per_page) - when 'issues' - issues.page(page).per(per_page) - when 'merge_requests' - merge_requests.page(page).per(per_page) - when 'milestones' - milestones.page(page).per(per_page) - else - Kaminari.paginate_array([]).page(page).per(per_page) - end + def objects(scope, page = nil, without_count = true) + collection = case scope + when 'projects' + projects.page(page).per(per_page) + when 'issues' + issues.page(page).per(per_page) + when 'merge_requests' + merge_requests.page(page).per(per_page) + when 'milestones' + milestones.page(page).per(per_page) + else + Kaminari.paginate_array([]).page(page).per(per_page) + end + + without_count ? collection.without_count : collection end def projects_count @@ -71,18 +73,46 @@ module Gitlab @milestones_count ||= milestones.count end + def limited_projects_count + @limited_projects_count ||= projects.limit(count_limit).count + end + + def limited_issues_count + return @limited_issues_count if @limited_issues_count + + # By default getting limited count (e.g. 1000+) is fast on issuable + # collections except for issues, where filtering both not confidential + # and confidential issues user has access to, is too complex. + # It's faster to try to fetch all public issues first, then only + # if necessary try to fetch all issues. + sum = issues(public_only: true).limit(count_limit).count + @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum + end + + def limited_merge_requests_count + @limited_merge_requests_count ||= merge_requests.limit(count_limit).count + end + + def limited_milestones_count + @limited_milestones_count ||= milestones.limit(count_limit).count + end + def single_commit_result? false end + def count_limit + 1001 + end + private def projects limit_projects.search(query) end - def issues - issues = IssuesFinder.new(current_user).execute + def issues(finder_params = {}) + issues = IssuesFinder.new(current_user, finder_params).execute unless default_project_filter issues = issues.where(project_id: project_ids_relation) end @@ -94,13 +124,13 @@ module Gitlab issues.full_search(query) end - issues.order('updated_at DESC') + issues.reorder('updated_at DESC') end def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) - milestones.order('updated_at DESC') + milestones.reorder('updated_at DESC') end def merge_requests @@ -115,7 +145,8 @@ module Gitlab else merge_requests.full_search(query) end - merge_requests.order('updated_at DESC') + + merge_requests.reorder('updated_at DESC') end def default_scope diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb new file mode 100644 index 00000000000..d01213bb6e0 --- /dev/null +++ b/lib/gitlab/setup_helper.rb @@ -0,0 +1,61 @@ +module Gitlab + module SetupHelper + class << self + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. + def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + + TOML.dump(config) + end + + # rubocop:disable Rails/Output + def create_gitaly_configuration(dir, force: false) + config_path = File.join(dir, 'config.toml') + FileUtils.rm_f(config_path) if force + + File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| + f.puts gitaly_configuration_toml(dir) + end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end + # rubocop:enable Rails/Output + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 40650fc5ee7..f4a41dc3eda 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,8 +99,12 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) + if url.start_with?('.', '/') + raise Error.new("don't use disk paths with import_repository: #{url.inspect}") + end + # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") success = cmd.import_project(url, git_timeout) @@ -122,11 +125,10 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) @@ -134,7 +136,10 @@ module Gitlab end end - # Move repository + # Move repository reroutes to mv_directory which is an alias for + # mv_namespace. Given the underlying implementation is a move action, + # indescriminate of what the folders might be. + # # storage - project's storage path # path - project disk path # new_path - new project disk path @@ -142,9 +147,11 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) - gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") + return false if path.empty? || new_path.empty? + + !!mv_directory(storage, "#{path}.git", "#{new_path}.git") end # Fork repository to new path @@ -156,13 +163,15 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") end - # Remove repository from file system + # Removes a repository from file system, using rm_diretory which is an alias + # for rm_namespace. Given the underlying implementation removes the name + # passed as second argument on the passed storage. # # storage - project's storage path # name - project disk path @@ -170,9 +179,14 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) - gitlab_projects(storage, "#{name}.git").rm_project + return false if name.empty? + + !!rm_directory(storage, "#{name}.git") + rescue ArgumentError => e + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + false end # Add new key to gitlab-shell @@ -181,6 +195,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'add-key', key_id, self.class.strip_key(key_content)]) end @@ -190,6 +206,8 @@ module Gitlab # Ex. # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) + return unless self.authorized_keys_enabled? + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| yield(KeyAdder.new(io)) end @@ -200,10 +218,11 @@ module Gitlab # Ex. # remove_key("key-342", "sha-rsa ...") # - def remove_key(key_id, key_content) + def remove_key(key_id, key_content = nil) + return unless self.authorized_keys_enabled? + args = [gitlab_shell_keys_path, 'rm-key', key_id] args << key_content if key_content - gitlab_shell_fast_execute(args) end @@ -213,15 +232,67 @@ module Gitlab # remove_all_keys # def remove_all_keys + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end + # Remove ssh keys from gitlab shell that are not in the DB + # + # Ex. + # remove_keys_not_found_in_db + # + def remove_keys_not_found_in_db + return unless self.authorized_keys_enabled? + + Rails.logger.info("Removing keys not found in DB") + + batch_read_key_ids do |ids_in_file| + ids_in_file.uniq! + keys_in_db = Key.where(id: ids_in_file) + + next unless ids_in_file.size > keys_in_db.count # optimization + + ids_to_remove = ids_in_file - keys_in_db.pluck(:id) + ids_to_remove.each do |id| + Rails.logger.info("Removing key-#{id} not found in DB") + remove_key("key-#{id}") + end + end + end + + # Iterate over all ssh key IDs from gitlab shell, in batches + # + # Ex. + # batch_read_key_ids { |batch| keys = Key.where(id: batch) } + # + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + list_key_ids do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + key_ids = lines.map { |l| l.chomp.to_i } + yield(key_ids) + end + end + end + + # Stream all ssh key IDs from gitlab shell, separated by newlines + # + # Ex. + # list_key_ids + # + def list_key_ids(&block) + return unless self.authorized_keys_enabled? + + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) + end + # Add empty directory for storing repositories # # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +314,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -255,13 +325,13 @@ module Gitlab rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end + alias_method :rm_directory, :rm_namespace # Move namespace directory inside repositories storage # # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled @@ -275,6 +345,7 @@ module Gitlab rescue GRPC::InvalidArgument false end + alias_method :mv_directory, :mv_namespace def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -334,6 +405,14 @@ module Gitlab File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + def authorized_keys_enabled? + # Return true if nil to ensure the authorized_keys methods work while + # fixing the authorized_keys file during migration. + return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil? + + Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + end + private def gitlab_projects(shard_path, disk_path) diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index b85f70e450e..4f86b3e8f73 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -16,7 +16,7 @@ module Gitlab when 'snippet_blobs' snippet_blobs.page(page).per(per_page) else - super + super(scope, nil, false) end end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb index 04bf1bf1d26..9b64c8e033a 100644 --- a/lib/gitlab/storage_check/cli.rb +++ b/lib/gitlab/storage_check/cli.rb @@ -59,9 +59,11 @@ module Gitlab if response.skipped_shards.any? warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" end + if response.failing_shards.any? warnings << "Failing shards: #{response.failing_shards.join(', ')}" end + logger.warn(warnings.join(' - ')) if warnings.any? end end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index 4a8e3c2eee0..53333b9b06b 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -37,12 +37,14 @@ module Gitlab def call(env) increment_active_requests + if block_requests? block_request(env) else sleep 0.2 if slow_requests? @app.call(env) end + ensure decrement_active_requests end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b290c716f97..76a1808c8ac 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -9,6 +9,7 @@ module Gitlab else block.call end + ensure model.record_timestamps = original_record_timestamps end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 961df0468a4..3b64cb32afa 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -12,6 +12,7 @@ module Gitlab puts "You are using the latest GitLab version" else puts "Newer GitLab version is available" + answer = if ARGV.first == "-y" "yes" else @@ -77,6 +78,7 @@ module Gitlab update_commands.each do |title, cmd| puts title puts " -> #{cmd.join(' ')}" + if system(env, *cmd) puts " -> OK" else diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index d9a5af09f08..f357488ac61 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -16,8 +16,10 @@ module Gitlab def can_do_action?(action) return false unless can_access_git? - @permission_cache ||= {} - @permission_cache[action] ||= user.can?(action, project) + permission_cache[action] = + permission_cache.fetch(action) do + user.can?(action, project) + end end def cannot_do_action?(action) @@ -88,6 +90,10 @@ module Gitlab private + def permission_cache + @permission_cache ||= {} + end + def can_access_git? user && user.can?(:access_git) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index b3baaf036d8..fa22f0e37b2 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -27,6 +27,10 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + def remove_line_breaks(str) + str.gsub(/\r?\n/, '') + end + def to_boolean(value) return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb new file mode 100644 index 00000000000..8bf6bcb1fe2 --- /dev/null +++ b/lib/gitlab/utils/override.rb @@ -0,0 +1,111 @@ +module Gitlab + module Utils + module Override + class Extension + def self.verify_class!(klass, method_name) + instance_method_defined?(klass, method_name) || + raise( + NotImplementedError.new( + "#{klass}\##{method_name} doesn't exist!")) + end + + def self.instance_method_defined?(klass, name, include_super: true) + klass.instance_methods(include_super).include?(name) || + klass.private_instance_methods(include_super).include?(name) + end + + attr_reader :subject + + def initialize(subject) + @subject = subject + end + + def add_method_name(method_name) + method_names << method_name + end + + def add_class(klass) + classes << klass + end + + def verify! + classes.each do |klass| + index = klass.ancestors.index(subject) + parents = klass.ancestors.drop(index + 1) + + method_names.each do |method_name| + parents.any? do |parent| + self.class.instance_method_defined?( + parent, method_name, include_super: false) + end || + raise( + NotImplementedError.new( + "#{klass}\##{method_name} doesn't exist!")) + end + end + end + + private + + def method_names + @method_names ||= [] + end + + def classes + @classes ||= [] + end + end + + # Instead of writing patterns like this: + # + # def f + # raise NotImplementedError unless defined?(super) + # + # true + # end + # + # We could write it like: + # + # extend ::Gitlab::Utils::Override + # + # override :f + # def f + # true + # end + # + # This would make sure we're overriding something. See: + # https://gitlab.com/gitlab-org/gitlab-ee/issues/1819 + def override(method_name) + return unless ENV['STATIC_VERIFICATION'] + + if is_a?(Class) + Extension.verify_class!(self, method_name) + else # We delay the check for modules + Override.extensions[self] ||= Extension.new(self) + Override.extensions[self].add_method_name(method_name) + end + end + + def included(base = nil) + return super if base.nil? # Rails concern, ignoring it + + super + + if base.is_a?(Class) # We could check for Class in `override` + # This could be `nil` if `override` was never called + Override.extensions[self]&.add_class(base) + end + end + + alias_method :prepended, :included + + def self.extensions + @extensions ||= {} + end + + def self.verify! + extensions.values.each(&:verify!) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5ab6cd5a4ef..633da44b22d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -34,7 +34,10 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + Gitlab::GitalyClient.feature_enabled?( + :post_receive_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) when 'git_upload_pack' true when 'info_refs' @@ -42,6 +45,7 @@ module Gitlab else raise "Unsupported action: #{action}" end + if feature_enabled params[:GitalyServer] = server end @@ -97,6 +101,9 @@ module Gitlab ) end + # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. + params['DisableCache'] = true if git_archive_cache_disabled? + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -244,6 +251,10 @@ module Gitlab right_commit_id: diff_refs.head_sha } end + + def git_archive_cache_disabled? + ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) + end end end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index b0563fb2d69..ff638c07755 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,4 +1,6 @@ require 'google/apis/container_v1' +require 'google/apis/cloudbilling_v1' +require 'google/apis/cloudresourcemanager_v1' module GoogleApi module CloudPlatform @@ -40,6 +42,22 @@ module GoogleApi true end + def projects_list + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + service.fetch_all(items: :projects) do |token| + service.list_projects(page_token: token, options: user_agent_header) + end + end + + def projects_get_billing_info(project_id) + service = Google::Apis::CloudbillingV1::CloudbillingService.new + service.authorization = access_token + + service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) + end + def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 8b145fb4511..d268f501b4a 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -66,6 +66,7 @@ module SystemCheck if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' + if check.repair! $stdout.puts 'Success'.color(:green) return diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index e65609d7001..4beb94eeb8e 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -7,4 +7,9 @@ namespace :dev do Rake::Task["gitlab:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke end + + desc "GitLab | Eager load application" + task load: :environment do + Rails.application.eager_load! + end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9dcf44fdc3e..2383bcf954b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -46,6 +46,7 @@ namespace :gitlab do puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) end + # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade $progress.puts 'Cleaning the database ... '.color(:blue) @@ -222,6 +223,7 @@ namespace :gitlab do task restore: :environment do $progress.puts "Restoring container registry images ... ".color(:blue) + if Gitlab.config.registry.enabled Backup::Registry.new.restore $progress.puts "done".color(:green) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dfade1f3885..a584eb97cf5 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -180,6 +180,7 @@ namespace :gitlab do puts "can't check, you have no projects".color(:magenta) return end + puts "" Project.find_each(batch_size: 100) do |project| @@ -210,6 +211,7 @@ namespace :gitlab do gitlab_shell_repo_base = gitlab_shell_path check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) puts "Running #{check_cmd}" + if system(check_cmd, chdir: gitlab_shell_repo_base) puts 'gitlab-shell self-check successful'.color(:green) else @@ -285,6 +287,7 @@ namespace :gitlab do return if process_count.zero? print 'Number of Sidekiq processes ... ' + if process_count == 1 puts '1'.color(:green) else @@ -387,14 +390,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, repository_storage| - namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) - - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - end - end + puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) + Rake::Task["gitlab:git:fsck"].execute end end @@ -430,7 +427,7 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for username? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| @@ -461,35 +458,4 @@ namespace :gitlab do puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end - - def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.color(:yellow)}" - - git_fsck(repo_dir) - check_config_lock(repo_dir) - check_ref_locks(repo_dir) - end - - def git_fsck(repo_dir) - puts "Running `git fsck`".color(:yellow) - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - if lock_files.present? - puts "Ref lock files exist:".color(:red) - lock_files.each do |lock_file| - puts " #{lock_file}" - end - else - puts "No ref lock files exist".color(:green) - end - end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index eb0f757aea7..04d56509ac6 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -84,6 +84,7 @@ namespace :gitlab do next unless user.ldap_user? print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." + if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) else diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index ba221e44e5d..77c28615856 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -14,6 +14,7 @@ namespace :gitlab do puts "Must specify a branch as an argument".color(:red) exit 1 end + args end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cf82134d97e..3f5dd2ae3b3 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -30,6 +30,20 @@ namespace :gitlab do end end + desc 'GitLab | Git | Check all repos integrity' + task fsck: :environment do + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| + check_config_lock(repo) + check_ref_locks(repo) + end + + if failures.empty? + puts "Done".color(:green) + else + output_failures(failures) + end + end + def perform_git_cmd(cmd, message) puts "Starting #{message} on all repositories" @@ -40,6 +54,8 @@ namespace :gitlab do else failures << repo end + + yield(repo) if block_given? end failures @@ -49,5 +65,24 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) + + if lock_files.present? + puts "Ref lock files exist:".color(:red) + + lock_files.each { |lock_file| puts " #{lock_file}" } + else + puts "No ref lock files exist".color(:green) + end + end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 4d880c05f99..a2e68c0471b 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -5,9 +5,11 @@ namespace :gitlab do require 'toml' warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version @@ -21,8 +23,8 @@ namespace :gitlab do command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? + Gitlab::SetupHelper.create_gitaly_configuration(args.dir) Dir.chdir(args.dir) do - create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? Bundler.with_original_env { run_command!(command) } @@ -39,60 +41,7 @@ namespace :gitlab do # Exclude gitaly-ruby configuration because that depends on the gitaly # installation directory. - puts gitaly_configuration_toml(gitaly_ruby: false) - end - - private - - # We cannot create config.toml files for all possible Gitaly configuations. - # For instance, if Gitaly is running on another machine then it makes no - # sense to write a config.toml file on the current machine. This method will - # only generate a configuration for the most common and simplest case: when - # we have exactly one Gitaly process and we are sure it is running locally - # because it uses a Unix socket. - # For development and testing purposes, an extra storage is added to gitaly, - # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_ruby: true) - storages = [] - address = nil - - Gitlab.config.repositories.storages.each do |key, val| - if address - if address != val['gitaly_address'] - raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." - end - elsif URI(val['gitaly_address']).scheme != 'unix' - raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." - else - address = val['gitaly_address'] - end - - storages << { name: key, path: val['path'] } - end - - if Rails.env.test? - storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } - end - - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } - config[:auth] = { token: 'secret' } if Rails.env.test? - config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby - config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path - - TOML.dump(config) - end - - def create_gitaly_configuration - File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml - end - rescue Errno::EEXIST - puts "Skipping config.toml generation:" - puts "A configuration file already exists." - rescue ArgumentError => e - puts "Skipping config.toml generation:" - puts e.message + puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) end end end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index b732db9db6e..d7f28691098 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -8,6 +8,7 @@ namespace :gitlab do namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end + scope.find_each do |project| base = File.join(project.repository_storage_path, project.disk_path) puts base + '.git' diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 0e6aed32c52..12ae4199b69 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -54,16 +54,6 @@ namespace :gitlab do # (Re)create hooks Rake::Task['gitlab:shell:create_hooks'].invoke - # Required for debian packaging with PKGR: Setup .ssh/environment with - # the current PATH, so that the correct ruby version gets loaded - # Requires to set "PermitUserEnvironment yes" in sshd config (should not - # be an issue since it is more than likely that there are no "normal" - # user accounts on a gitlab server). The alternative is for the admin to - # install a ruby (1.9.3+) in the global path. - File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f| - f.puts "PATH=#{ENV['PATH']}" - end - Gitlab::Shell.ensure_secret_token! end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 6723662703c..c1182af1014 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -130,7 +130,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index f44abc2b81b..a25f7ce59c7 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -10,6 +10,7 @@ namespace :gitlab do puts "This rake task is not meant fo production instances".red exit(1) end + admin = User.find_by(admin: true) unless admin diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake new file mode 100644 index 00000000000..df31567ce64 --- /dev/null +++ b/lib/tasks/gitlab/uploads.rake @@ -0,0 +1,44 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + puts 'Checking integrity of uploaded files' + + uploads_batches do |batch| + batch.each do |upload| + puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) + + if upload.exist? + check_checksum(upload) + else + puts " * File does not exist on the file system".color(:red) + end + end + end + + puts 'Done!' + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def calculate_checksum(absolute_path) + Digest::SHA256.file(absolute_path).hexdigest + end + + def check_checksum(upload) + checksum = calculate_checksum(upload.absolute_path) + + if checksum != upload.checksum + puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) + end + end + + def uploads_batches(&block) + Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + yield relation + end + end + end +end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index e7ac0b5859f..308ffb0e284 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -3,9 +3,11 @@ namespace :gitlab do desc "GitLab | Install or upgrade gitlab-workhorse" task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 7b63e93db0e..3ab406eff2c 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -1,5 +1,17 @@ unless Rails.env.production? namespace :lint do + task :static_verification_env do + ENV['STATIC_VERIFICATION'] = 'true' + end + + desc "GitLab | lint | Static verification" + task static_verification: %w[ + lint:static_verification_env + dev:load + ] do + Gitlab::Utils::Override.verify! + end + desc "GitLab | lint | Lint JavaScript files using ESLint" task :javascript do Rake::Task['eslint'].invoke diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index fc2cea8c016..aa2d01730d7 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -4,6 +4,7 @@ task migrate_iids: :environment do Issue.where(iid: nil).find_each(batch_size: 100) do |issue| begin issue.set_iid + if issue.update_attribute(:iid, issue.iid) print '.' else @@ -19,6 +20,7 @@ task migrate_iids: :environment do MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr| begin mr.set_iid + if mr.update_attribute(:iid, mr.iid) print '.' else @@ -34,6 +36,7 @@ task migrate_iids: :environment do Milestone.where(iid: nil).find_each(batch_size: 100) do |m| begin m.set_iid + if m.update_attribute(:iid, m.iid) print '.' else diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index c9e3eed82f2..c996537cfbe 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -7,6 +7,7 @@ require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') +require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do @@ -17,4 +18,5 @@ task setup_postgresql: :environment do AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up + ReworkRedirectRoutesIndexes.new.up end |