diff options
Diffstat (limited to 'lib')
120 files changed, 1550 insertions, 874 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index d767af36e8e..efcf0976a81 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,6 +2,8 @@ module API class API < Grape::API include APIGuard + allow_access_with_scope :api + version %w(v3 v4), using: :path version 'v3', using: :path do @@ -44,7 +46,6 @@ module API mount ::API::V3::Variables end - before { allow_access_with_scope :api } before { header['X-Frame-Options'] = 'SAMEORIGIN' } before { Gitlab::I18n.locale = current_user&.preferred_language } diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9fcf04efa38..0d2d71e336a 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -23,6 +23,23 @@ module API install_error_responders(base) end + class_methods do + # Set the authorization scope(s) allowed for an API endpoint. + # + # A call to this method maps the given scope(s) to the current API + # endpoint class. If this method is called multiple times on the same class, + # the scopes are all aggregated. + def allow_access_with_scope(scopes, options = {}) + Array(scopes).each do |scope| + allowed_scopes << Scope.new(scope, options) + end + end + + def allowed_scopes + @scopes ||= [] + end + end + # Helper Methods for Grape Endpoint module HelperMethods # Invokes the doorkeeper guard. @@ -47,7 +64,7 @@ module API access_token = find_access_token return nil unless access_token - case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) @@ -74,18 +91,6 @@ module API @current_user end - # Set the authorization scope(s) allowed for the current request. - # - # Note: A call to this method adds to any previous scopes in place. This is done because - # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then - # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the - # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they - # need to be stored. - def allow_access_with_scope(*scopes) - @scopes ||= [] - @scopes.concat(scopes.map(&:to_s)) - end - private def find_user_by_authentication_token(token_string) @@ -96,7 +101,7 @@ module API access_token = PersonalAccessToken.active.find_by_token(token_string) return unless access_token - if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes) User.find(access_token.user_id) end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cef5a0abe12..09a88869063 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -112,6 +112,7 @@ module API expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs + expose :ci_config_path expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end @@ -254,7 +255,7 @@ module API class ProjectEntity < Grape::Entity expose :id, :iid - expose(:project_id) { |entity| entity.project.id } + expose(:project_id) { |entity| entity&.project.try(:id) } expose :title, :description expose :state, :created_at, :updated_at end @@ -266,7 +267,12 @@ module API expose :deleted_file?, as: :deleted_file end - class Milestone < ProjectEntity + class Milestone < Grape::Entity + expose :id, :iid + expose(:project_id) { |entity| entity&.project_id } + expose(:group_id) { |entity| entity&.group_id } + expose :title, :description + expose :state, :created_at, :updated_at expose :due_date expose :start_date end @@ -308,12 +314,35 @@ module API expose :id end + class MergeRequestSimple < ProjectEntity + expose :title + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end + end + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].upvotes + else + merge_request.upvotes + end + end + expose :downvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].downvotes + else + merge_request.downvotes + end + end expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :label_names, as: :labels + expose :labels do |merge_request, options| + # Avoids an N+1 query since labels are preloaded + merge_request.labels.map(&:title).sort + end expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds @@ -434,7 +463,7 @@ module API target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? - Gitlab::Application.routes.url_helpers.public_send(target_url, + Gitlab::Routing.url_helpers.public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) end @@ -502,12 +531,20 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:current_user].id) + if options.key?(:project_members) + (options[:project_members] || []).find { |member| member.source_id == project.id } + else + project.project_members.find_by(user_id: options[:current_user].id) + end end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - project.group.group_members.find_by(user_id: options[:current_user].id) + if options.key?(:group_members) + (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } + else + project.group.group_members.find_by(user_id: options[:current_user].id) + end end end end @@ -584,7 +621,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text @@ -831,7 +869,7 @@ module API end class Cache < Grape::Entity - expose :key, :untracked, :paths + expose :key, :untracked, :paths, :policy end class Credentials < Grape::Entity @@ -874,5 +912,11 @@ module API expose :dependencies, using: Dependency end end + + class UserAgentDetail < Grape::Entity + expose :user_agent + expose :ip_address + expose :submitted, as: :akismet_submitted + end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 21745916463..9385c6ca174 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -14,14 +14,12 @@ module API end end - def gate_target(params) - if params[:feature_group] - Feature.group(params[:feature_group]) - elsif params[:user] - User.find_by_username(params[:user]) - else - gate_value(params) - end + def gate_targets(params) + targets = [] + targets << Feature.group(params[:feature_group]) if params[:feature_group] + targets << User.find_by_username(params[:user]) if params[:user] + + targets end end @@ -42,18 +40,25 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' - mutually_exclusive :feature_group, :user end post ':name' do feature = Feature.get(params[:name]) - target = gate_target(params) + targets = gate_targets(params) value = gate_value(params) case value when true - feature.enable(target) + if targets.present? + targets.each { |target| feature.enable(target) } + else + feature.enable + end when false - feature.disable(target) + if targets.present? + targets.each { |target| feature.disable(target) } + else + feature.disable + end else feature.enable_percentage_of_time(value) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2c73a6fdc4e..0f4791841d2 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -268,6 +268,7 @@ module API finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] finder_params[:archived] = params[:archived] finder_params[:search] = params[:search] if params[:search] + finder_params[:user] = params.delete(:user) if params[:user] finder_params end @@ -313,7 +314,7 @@ module API def present_artifacts!(artifacts_file) return not_found! unless artifacts_file.exists? - + if artifacts_file.file_storage? present_file!(artifacts_file.path, artifacts_file.filename) else @@ -342,8 +343,8 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: @scopes) - @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) + @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) @initial_current_user ||= find_user_from_warden unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? @@ -407,5 +408,22 @@ module API exception.status == 500 end + + # An array of scopes that were registered (using `allow_access_with_scope`) + # for the current endpoint class. It also returns scopes registered on + # `API::API`, since these are meant to apply to all API routes. + def scopes_registered_for_endpoint + @scopes_registered_for_endpoint ||= + begin + endpoint_classes = [options[:for].presence, ::API::API].compact + endpoint_classes.reduce([]) do |memo, endpoint| + if endpoint.respond_to?(:allowed_scopes) + memo.concat(endpoint.allowed_scopes) + else + memo + end + end + end + end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 5e9cf5e68b1..ecb79317093 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,6 +1,11 @@ module API module Helpers module InternalHelpers + SSH_GITALY_FEATURES = { + 'git-receive-pack' => :ssh_receive_pack, + 'git-upload-pack' => :ssh_upload_pack + }.freeze + def wiki? set_project unless defined?(@wiki) @wiki @@ -10,7 +15,7 @@ module API set_project unless defined?(@project) @project end - + def redirected_path @redirected_path end @@ -54,15 +59,33 @@ module API Gitlab::GlRepository.gl_repository(project, wiki?) end - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - def repository_path + # Return the repository depending on whether we want the wiki or the + # regular repository + def repository if wiki? - project.wiki.repository.path_to_repo + project.wiki.repository else - project.repository.path_to_repo + project.repository end end + + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + def repository_path + repository.path_to_repo + end + + # Return the Gitaly Address if it is enabled + def gitaly_payload(action) + feature = SSH_GITALY_FEATURES[action] + return unless feature && Gitlab::GitalyClient.feature_enabled?(feature) + + { + repository: repository.gitaly_repository, + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index f1c79970ba4..465363da582 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -47,7 +47,8 @@ module API { status: true, gl_repository: gl_repository, - repository_path: repository_path + repository_path: repository_path, + gitaly: gitaly_payload(params[:action]) } end @@ -100,7 +101,7 @@ module API end get "/broadcast_message" do - if message = BroadcastMessage.current.last + if message = BroadcastMessage.current&.last present message, with: Entities::BroadcastMessage else {} diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 09dca0dff8b..64be08094ed 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -241,6 +241,22 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + + desc 'Get the user agent details for an issue' do + success Entities::UserAgentDetail + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ":id/issues/:issue_iid/user_agent_detail" do + authenticated_as_admin! + + issue = find_project_issue(params[:issue_iid]) + + return not_found!('UserAgentDetail') unless issue.user_agent_detail + + present issue.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 1118fc7465b..ac33b2b801c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -10,6 +10,8 @@ module API resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints + helpers ::Gitlab::IssuableMetadata + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -41,9 +43,15 @@ module API args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) - merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + merge_requests = MergeRequestsFinder.new(current_user, args).execute + .reorder(args[:order_by] => args[:sort]) + merge_requests = paginate(merge_requests) + .preload(:target_project) + + return merge_requests if args[:view] == 'simple' - merge_requests.reorder(args[:order_by] => args[:sort]) + merge_requests + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) end params :optional_params_ce do @@ -74,6 +82,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' use :pagination end get ":id/merge_requests" do @@ -81,7 +90,17 @@ module API merge_requests = find_merge_requests(project_id: user_project.id) - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + options = { with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options end desc 'Create a merge request' do diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 93d89209934..dbeaf9e17ef 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -74,9 +74,10 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :update_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -92,9 +93,10 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :update_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -110,9 +112,10 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :admin_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :admin_pipeline_schedule, pipeline_schedule status :accepted present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 64efe82a937..3320eadff0d 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -131,6 +131,22 @@ module API content_type 'text/plain' present snippet.content end + + desc 'Get the user agent details for a project snippet' do + success Entities::UserAgentDetail + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id/user_agent_detail" do + authenticated_as_admin! + + snippet = Snippet.find_by!(id: params[:id]) + + return not_found!('UserAgentDetail') unless snippet.user_agent_detail + + present snippet.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d0bd64b2972..c459257158d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -10,6 +10,7 @@ module API helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' @@ -35,61 +36,86 @@ module API params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end - end - resource :projects do - helpers do - params :collection_params do - use :sort_params - use :filter_params - use :pagination - - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - end + params :collection_params do + use :sort_params + use :filter_params + use :pagination - params :sort_params do - optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], - default: 'created_at', desc: 'Return projects ordered by field' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return projects sorted in ascending and descending order' - end + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end - params :filter_params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, - desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of projects matching the search criteria' - optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' - optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' - optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' - optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' - end + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end - params :create_params do - optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' - optional :import_url, type: String, desc: 'URL from which the project is imported' - end + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of projects matching the search criteria' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end - def present_projects(options = {}) - projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute - projects = reorder_projects(projects) - projects = projects.with_statistics if params[:statistics] - projects = projects.with_issues_enabled if params[:with_issues_enabled] - projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] - - options = options.reverse_merge( - with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, - statistics: params[:statistics], - current_user: current_user - ) - options[:with] = Entities::BasicProjectDetails if params[:simple] - - present paginate(projects), options + def present_projects(options = {}) + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + + if current_user + projects = projects.includes(:route, :taggings, namespace: :route) + project_members = current_user.project_members + group_members = current_user.group_members end + + options = options.reverse_merge( + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + project_members: project_members, + group_members: group_members, + current_user: current_user + ) + options[:with] = Entities::BasicProjectDetails if params[:simple] + + present paginate(projects), options end + end + resource :users, requirements: { user_id: %r{[^/]+} } do + desc 'Get a user projects' do + success Entities::BasicProjectDetails + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :collection_params + use :statistics_params + end + get ":user_id/projects" do + user = find_user(params[:user_id]) + not_found!('User') unless user + + params[:user] = user + + present_projects + end + end + + resource :projects do desc 'Get a list of visible projects for authenticated user' do success Entities::BasicProjectDetails end diff --git a/lib/api/scope.rb b/lib/api/scope.rb new file mode 100644 index 00000000000..d5165b2e482 --- /dev/null +++ b/lib/api/scope.rb @@ -0,0 +1,23 @@ +# Encapsulate a scope used for authorization, such as `api`, or `read_user` +module API + class Scope + attr_reader :name, :if + + def initialize(name, options = {}) + @name = name.to_sym + @if = options[:if] + end + + # Are the `scopes` passed in sufficient to adequately authorize the passed + # request for the scope represented by the current instance of this class? + def sufficient?(scopes, request) + scopes.include?(self.name) && verify_if_condition(request) + end + + private + + def verify_if_condition(request) + self.if.nil? || self.if.call(request) + end + end +end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d598f9a62a2..b19095d1252 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -65,6 +65,7 @@ module API :shared_runners_enabled, :sidekiq_throttling_enabled, :sign_in_text, + :password_authentication_enabled, :signin_enabled, :signup_enabled, :terminal_max_session_time, @@ -95,7 +96,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -176,6 +179,10 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c630c24c339..fd634037a77 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -140,6 +140,22 @@ module API content_type 'text/plain' present snippet.content end + + desc 'Get the user agent details for a snippet' do + success Entities::UserAgentDetail + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/user_agent_detail" do + authenticated_as_admin! + + snippet = Snippet.find_by!(id: params[:id]) + + return not_found!('UserAgentDetail') unless snippet.user_agent_detail + + present snippet.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 5b9d9a71be4..81c68ea2658 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,10 +1,9 @@ module API class Users < Grape::API include PaginationParams + include APIGuard - before do - allow_access_with_scope :read_user if request.get? - end + allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do before do @@ -49,6 +48,8 @@ module API optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + optional :created_after, type: DateTime, desc: 'Return users created after the specified time' + optional :created_before, type: DateTime, desc: 'Return users created before the specified time' all_or_none_of :extern_uid, :provider use :pagination @@ -56,6 +57,10 @@ module API get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + unless current_user&.admin? + params.except!(:created_after, :created_before) + end + users = UsersFinder.new(current_user, params).execute authorized = can?(current_user, :read_users_list) @@ -416,7 +421,16 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic + entity = + if sudo? + Entities::UserWithPrivateDetails + elsif current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end + + present current_user, with: entity end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c848f52723b..3759250f7f6 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -161,7 +161,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 748d6b97d4f..202011cfcbe 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,7 +44,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -116,7 +118,7 @@ module API :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, :user_oauth_applications, :user_default_external, :signup_enabled, :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication, :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, @@ -126,7 +128,13 @@ module API :housekeeping_enabled, :terminal_max_session_time end put "application/settings" do - if current_settings.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 37020019e07..cf106f2552d 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -2,9 +2,11 @@ module API module V3 class Users < Grape::API include PaginationParams + include APIGuard + + allow_access_with_scope :read_user, if: -> (request) { request.get? } before do - allow_access_with_scope :read_user if request.get? authenticate! end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 10374995497..7fa528fb2d3 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -45,7 +45,9 @@ module API optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do - variable = user_project.variables.create(declared_params(include_missing: false)) + variable_params = declared_params(include_missing: false) + + variable = user_project.variables.create(variable_params) if variable.valid? present variable, with: Entities::Variable @@ -67,7 +69,9 @@ module API return not_found!('Variable') unless variable - if variable.update(declared_params(include_missing: false).except(:key)) + variable_params = declared_params(include_missing: false).except(:key) + + if variable.update(variable_params) present variable, with: Entities::Variable else render_validation_error!(variable) diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index eaacb9591b1..21bcb1c5ca8 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -30,7 +30,7 @@ module Banzai def url_for_object(range, project) h = Gitlab::Routing.url_helpers - h.namespace_project_compare_url(project.namespace, project, + h.project_compare_url(project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 69c06117eda..714e0319025 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - h.namespace_project_commit_url(project.namespace, project, commit, + h.project_commit_url(project, commit, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a605dea149e..5364984c9d3 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -61,8 +61,7 @@ module Banzai def url_for_object(label, project) h = Gitlab::Routing.url_helpers - h.namespace_project_issues_url(project.namespace, project, label_name: label.name, - only_path: context[:only_path]) + h.project_issues_url(project, label_name: label.name, only_path: context[:only_path]) end def object_link_text(object, matches) diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 3888acf935e..0eab865ac04 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -17,7 +17,7 @@ module Banzai def url_for_object(mr, project) h = Gitlab::Routing.url_helpers - h.namespace_project_merge_request_url(project.namespace, project, mr, + h.project_merge_request_url(project, mr, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index f12014e191f..45c033d32a8 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -49,7 +49,7 @@ module Banzai def url_for_object(milestone, project) h = Gitlab::Routing.url_helpers - h.namespace_project_milestone_url(project.namespace, project, milestone, + h.project_milestone_url(project, milestone, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 6640168bfa2..a6f8650ed3d 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -30,6 +30,8 @@ module Banzai attributes = attributes.reject { |_, v| v.nil? } attributes[:reference_type] ||= self.class.reference_type + attributes[:container] ||= 'body' + attributes[:placement] ||= 'bottom' attributes.delete(:original) if context[:no_original_data] attributes.map do |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 212a0bbf2a0..134a192c22b 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -17,7 +17,7 @@ module Banzai def url_for_object(snippet, project) h = Gitlab::Routing.url_helpers - h.namespace_project_snippet_url(project.namespace, project, snippet, + h.project_snippet_url(project, snippet, only_path: context[:only_path]) end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index a798927823f..f3356d6c51e 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -107,7 +107,7 @@ module Banzai if author && !project.team.member?(author) link_content else - url = urls.namespace_project_url(project.namespace, project, + url = urls.project_url(project, only_path: context[:only_path]) data = data_attribute(project: project.id, author: author.try(:id)) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 56ad2c77c7d..cf3a0336792 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -80,6 +80,8 @@ module Ci artifacts: job[:artifacts], cache: job[:cache], dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], after_script: job[:after_script], environment: job[:environment] }.compact } diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index dd864eea3fa..721ed97bb6b 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -126,8 +126,7 @@ module ExtractsPath raise InvalidPathError unless @commit @hex_path = Digest::SHA1.hexdigest(@path) - @logs_path = logs_file_namespace_project_ref_path(@project.namespace, - @project, @ref, @path) + @logs_path = logs_file_project_ref_path(@project, @ref, @path) rescue RuntimeError, NoMethodError, InvalidPathError render_404 diff --git a/lib/feature.rb b/lib/feature.rb index 363f66ba60e..4bd29aed687 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -57,5 +57,11 @@ class Feature Flipper.new(adapter) end end + + # This method is called from config/initializers/flipper.rb and can be used + # to register Flipper groups. + # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups + def register_feature_groups + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3933c3b04dd..9bed81e7327 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,7 +37,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -48,6 +48,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -130,13 +134,13 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES) Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_scoped_token?(token, ["api"]) + token && token.accessible? && valid_scoped_token?(token, [:api]) end def valid_scoped_token?(token, scopes) diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d95ecd7b291..b0741b1fba7 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,24 +1,45 @@ module Gitlab module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue - .new(BackgroundMigrationWorker.sidekiq_options['queue']) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - queue.each do |job| - migration_class, migration_args = job.args + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args - next unless migration_class == steal_class + next unless job.queue == self.queue + next unless migration_class == steal_class - perform(migration_class, migration_args) + begin + perform(migration_class, migration_args, retries: 3) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) - job.delete + raise + end + end end end + ## + # Performs a background migration. + # # class_name - The name of the background migration class as defined in the # Gitlab::BackgroundMigration namespace. # diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb new file mode 100644 index 00000000000..91540127ea9 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -0,0 +1,19 @@ +module Gitlab + module BackgroundMigration + class MigrateBuildStageIdReference + def perform(start_id, stop_id) + sql = <<-SQL.strip_heredoc + UPDATE ci_builds + SET stage_id = + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND ci_builds.stage_id IS NULL + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb index f87a7b7942e..2ee35a0d4c1 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/build/metadata.rb @@ -15,12 +15,11 @@ module Gitlab end def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) + build_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, id: @ref) end end end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb index 53588185622..e898f5d790e 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -16,13 +16,11 @@ module Gitlab end def image_url - coverage_namespace_project_badges_url(@project.namespace, - @project, @ref, - format: :svg) + coverage_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, @ref) end end end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 4a049ef758d..8ad6f3cb986 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -4,7 +4,7 @@ module Gitlab # Abstract class for badge metadata # class Metadata - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing include ActionView::Helpers::AssetTagHelper include ActionView::Helpers::UrlHelper diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f074df9c7a1..d7e09acbbf3 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,11 +7,14 @@ module Gitlab # class Cache < Node include Configurable + include Attributable - ALLOWED_KEYS = %i[key untracked paths].freeze + ALLOWED_KEYS = %i[key untracked paths policy].freeze + DEFAULT_POLICY = 'pull-push'.freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true end entry :key, Entry::Key, @@ -25,8 +28,15 @@ module Gitlab helpers :key + attributes :policy + def value - super.merge(key: key_value) + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + + result end end end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 439ef0ce015..8ad3e57e59d 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,9 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + cancel_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index b173c23fba4..c0c7c7f5b5d 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + project_job_path(subject.project, subject) end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index e80f3263794..c7726543599 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 56303e4cb17..8c8fdc56d75 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,9 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + retry_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 2778d6f3b52..d464738deaf 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 76bfd18bf40..61bb07beb0f 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject) + project_pipeline_path(subject.project, subject) end def has_action? diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index 7852f492e1d..bc99d925347 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -8,10 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject.pipeline, - anchor: subject.name) + project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) end def has_action? diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 75a213ef752..98dfe900044 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,7 +1,7 @@ module Gitlab module Conflict class File - include Gitlab::Routing.url_helpers + include Gitlab::Routing include IconsHelper MissingResolution = Class.new(ResolutionError) @@ -205,9 +205,7 @@ module Gitlab old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), - blob_path: namespace_project_blob_path(merge_request.project.namespace, - merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)) + blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path)) } json_hash.tap do |json_hash| @@ -223,11 +221,10 @@ module Gitlab end def content_path - conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, - merge_request.project, - merge_request, - old_path: their_path, - new_path: our_path) + conflict_for_path_project_merge_request_path(merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 818b3d9c46b..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available end end @@ -33,12 +33,7 @@ module Gitlab def uncached_application_settings return fake_application_settings unless connect_to_db? - # This loads from the database into the cache, so handle Redis errors - begin - db_settings = ::ApplicationSetting.current - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - end + db_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 9d25ef078e8..f5d08c0b658 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -13,6 +13,10 @@ module Gitlab MergeRequestDiff.arel_table end + def mr_diff_commits_table + MergeRequestDiffCommit.arel_table + end + def mr_closing_issues_table MergeRequestsClosingIssues.arel_table end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7d342a2d2cb..b260822788d 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -2,40 +2,59 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @projections = [mr_diff_table[:st_commits].as('commits'), + @projections = [mr_diff_table[:id], + mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end def events_query - base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + base_query + .join(mr_diff_table) + .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) super end private + def merge_request_diff_commits + @merge_request_diff_commits ||= + MergeRequestDiffCommit + .where(merge_request_diff_id: event_result.map { |event| event['id'] }) + .group_by(&:merge_request_diff_id) + end + def serialize(event) - st_commit = first_time_reference_commit(event.delete('commits'), event) + commit = first_time_reference_commit(event) - return unless st_commit + return unless commit - serialize_commit(event, st_commit, query) + serialize_commit(event, commit, query) end - def first_time_reference_commit(commits, event) + def first_time_reference_commit(event) + return nil unless event && merge_request_diff_commits + + commits = + if event['st_commits'].present? + YAML.load(event['st_commits']) + else + merge_request_diff_commits[event['id'].to_i] + end + return nil if commits.blank? - YAML.load(commits).find do |commit| + commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i end end - def serialize_commit(event, st_commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + def serialize_commit(event, commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 1a5887dab7e..edd7795eef0 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -337,7 +337,7 @@ module Gitlab # In the EE repo $ git push origin #{ee_branch_prefix} - ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. Once this is done, you can retry this failed build, and it should pass. diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index ea035e33eff..dd1d9dcd555 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -4,7 +4,7 @@ module Gitlab class RepositoryPush attr_reader :author_id, :ref, :action - include Gitlab::Routing.url_helpers + include Gitlab::Routing include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project @@ -96,20 +96,13 @@ module Gitlab def target_url if @action == :push && commits if commits.length > 1 - namespace_project_compare_url(project_namespace, - project, - from: compare.start_commit, - to: compare.head_commit) + project_compare_url(project, from: compare.start_commit, to: compare.head_commit) else - namespace_project_commit_url(project_namespace, - project, - commits.first) + project_commit_url(project, commits.first) end else unless @action == :delete - namespace_project_tree_url(project_namespace, - project, - ref_name) + project_tree_url(project, ref_name) end end end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a0f46594eb1..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -26,17 +26,17 @@ module Gitlab EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -45,24 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..0deaab01b5b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC for #load_blame. + module Gitlab module Git class Blame @@ -24,6 +26,7 @@ module Gitlab private + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) # Read in binary mode to ensure ASCII-8BIT diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index a7aceab4c14..b6dd3cd20e0 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -41,10 +43,6 @@ module Gitlab commit_id: sha ) when :BLOB - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data) new( id: entry.oid, name: name, @@ -53,7 +51,7 @@ module Gitlab mode: entry.mode.to_s(8), path: path, commit_id: sha, - binary: detect && detect[:type] == :binary + binary: binary?(entry.data) ) end end @@ -87,16 +85,32 @@ module Gitlab end def raw(repository, sha) - blob = repository.lookup(sha) + Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + else + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + end + end - new( - id: blob.oid, - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - binary: blob.binary? - ) + def binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary end + private + # Recursive search of blob id by path # # Ex. @@ -165,8 +179,17 @@ module Gitlab return if @data == '' # don't mess with submodule blobs return @data if @loaded_all_data + Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + @data = begin + if is_enabled + Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data + else + repository.lookup(id).content + end + end + end + @loaded_all_data = true - @data = repository.lookup(id).content @loaded_size = @data.bytesize @binary = nil end @@ -175,6 +198,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def truncated? size && (size > loaded_size) end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 9c0606d780a..d0f04d25db2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -124,6 +125,7 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) actual_options = options.dup @@ -243,6 +245,8 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff diff_from_parent.patch end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 78e440395a5..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index cf7829a583b..cf95f673667 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -81,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 555894907cc..0d8fe185ac5 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index f4e3b5e5129..4a43b9b444d 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class GitmodulesParser diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 5042916343b..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 1add037fa5f..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index @@ -110,10 +114,6 @@ module Gitlab if segment == '..' raise IndexError, 'Path cannot include directory traversal' end - - unless segment =~ Gitlab::Regex.file_name_regex - raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" - end end pathname.to_s diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 23d0c8a9bdb..5aefa1404ad 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -80,16 +80,10 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389 + def branches(sort_by: nil) + branches_filter(sort_by: sort_by) end def reload_rugged @@ -107,7 +101,10 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) @@ -115,7 +112,7 @@ module Gitlab if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -162,6 +159,8 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags rugged.references.each("refs/tags/*").map do |ref| message = nil @@ -174,7 +173,8 @@ module Gitlab end end - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) end.sort_by(&:name) end @@ -204,13 +204,6 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end @@ -297,28 +290,6 @@ module Gitlab (size.to_f / 1024).round(2) end - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) - end - - greps - end - # Use the Rugged Walker API to build an array of commits. # # Usage. @@ -331,85 +302,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -454,7 +350,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -553,23 +449,35 @@ module Gitlab # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) - if submodules(ref).any? - submodule = submodules(ref)[path] - - if submodule - submodule['url'] + Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| + if is_enabled + gitaly_submodule_url_for(ref, path) + else + if submodules(ref).any? + submodule = submodules(ref)[path] + submodule['url'] if submodule + end end end end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + gitaly_migrate(:commit_count) do |is_enabled| + if is_enabled + gitaly_commit_client.commit_count(ref) + else + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + end end # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or @@ -770,7 +678,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -829,6 +738,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -855,6 +765,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -898,6 +809,103 @@ module Gitlab private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 @@ -915,6 +923,18 @@ module Gitlab fill_submodule_ids(commit, parser.parse) end + def gitaly_submodule_url_for(ref, path) + # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. + commit_object = gitaly_commit_client.tree_entry(ref, path, 1) + + return unless commit_object && commit_object.type == :COMMIT + + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + found_module = GitmodulesParser.new(gitmodules.data).parse[path] + + found_module && found_module['url'] + end + def alternate_object_directories Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact end @@ -1057,73 +1077,6 @@ module Gitlab index end - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b9afa05c819..8122ff0e81f 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,6 +12,8 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' @@ -40,6 +44,8 @@ module Gitlab end end + private + # Recursive search of tree id for path # # Ex. @@ -80,6 +86,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def dir? type == :tree end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f605c06dfc3..197a94487eb 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -70,12 +70,8 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.enabled? - Gitlab.config.gitaly.enabled - end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) - return false if !enabled? || status == MigrationStatus::DISABLED + return false if status == MigrationStatus::DISABLED feature = Feature.get("gitaly_#{feature}") diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob.rb new file mode 100644 index 00000000000..0c398b46a08 --- /dev/null +++ b/lib/gitlab/gitaly_client/blob.rb @@ -0,0 +1,30 @@ +module Gitlab + module GitalyClient + class Blob + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + end + + def get_blob(oid:, limit:) + request = Gitaly::GetBlobRequest.new( + repository: @gitaly_repo, + oid: oid, + limit: limit + ) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + + blob = response.first + return unless blob.oid.present? + + data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + + Gitlab::Git::Blob.new( + id: blob.oid, + size: blob.size, + data: data, + binary: Gitlab::Git::Blob.binary?(data) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index b8877619797..aafc0520664 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -56,6 +56,15 @@ module Gitlab entry end + def commit_count(ref) + request = Gitaly::CountCommitsRequest.new( + repository: @gitaly_repo, + revision: ref + ) + + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index 6edc69de078..2c17b67d4b7 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -72,11 +72,39 @@ module Gitlab Gitlab::Git::Branch.new( @repository, encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) ) end end end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name, + author_email: response.commit_author.email, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name, + committer_email: response.commit_committer.email + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index e78b7f22e03..bebde857b16 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -35,9 +35,9 @@ module Gitlab repository_storages.flat_map do |storage_name| tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name), + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name), + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name) ].flatten end end @@ -52,7 +52,7 @@ module Gitlab ] end rescue RuntimeError => ex - Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..3dcb28a193c 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -20,7 +20,7 @@ module Gitlab [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) + metric("#{metric_prefix}_latency_seconds", elapsed) ] end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1860352c96d..c8ad3a7a5e0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,7 @@ project_tree: - :author - :events - merge_request_diff: + - :merge_request_diff_commits - :merge_request_diff_files - :events - :timelogs diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index c56c1a4322f..cdbdfa10d0e 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -76,5 +76,44 @@ module Gitlab url.to_s end + + def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + config = { + apiVersion: 'v1', + clusters: [ + name: 'gitlab-deploy', + cluster: { + server: url + } + ], + contexts: [ + name: 'gitlab-deploy', + context: { + cluster: 'gitlab-deploy', + namespace: namespace, + user: 'gitlab-deploy' + } + ], + 'current-context': 'gitlab-deploy', + kind: 'Config', + users: [ + { + name: 'gitlab-deploy', + user: { token: token } + } + ] + } + + kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem + + config.deep_stringify_keys + end + + private + + def kubeconfig_embed_ca_pem(config, ca_pem) + cluster = config.dig(:clusters, 0, :cluster) + cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb deleted file mode 100644 index b3da360be8f..00000000000 --- a/lib/gitlab/metrics/connection_rack_middleware.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module Metrics - class ConnectionRackMiddleware - def initialize(app) - @app = app - end - - def self.rack_request_count - @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count') - end - - def self.rack_response_count - @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count') - end - - def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count') - end - - def self.rack_execution_time - @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10]) - end - - def call(env) - method = env['REQUEST_METHOD'].downcase - started = Time.now.to_f - begin - ConnectionRackMiddleware.rack_request_count.increment(method: method) - - status, headers, body = @app.call(env) - - ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status) - [status, headers, body] - rescue - ConnectionRackMiddleware.rack_uncaught_errors_count.increment - raise - ensure - elapsed = Time.now.to_f - started - ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed) - end - end - end - end -end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b3f453e506d..3f2bbd9f6a6 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -101,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 0d541935bc6..22332474945 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -34,7 +34,7 @@ module Gitlab write_csv do |csv| ActiveRecord::Base.transaction do - User.with_two_factor.in_batches do |relation| + User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) rows.each do |row| user = %i[id ciphertext iv salt].zip(row).to_h diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 10eb99fb461..d81f825ef96 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -112,6 +112,7 @@ module Gitlab # this group would not be accessible through `/groups/parent/activity` since # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ + - activity analytics audit_events diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 163a40ad306..56112ec2301 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,34 @@ module Gitlab module PerformanceBar - def self.enabled? - Feature.enabled?('gitlab_performance_bar') + include Gitlab::CurrentSettings + + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes + + def self.enabled?(user = nil) + return false unless user && allowed_group_id + + allowed_user_ids.include?(user.id) + end + + def self.allowed_group_id + current_application_settings.performance_bar_allowed_group_id + end + + def self.allowed_user_ids + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do + group = Group.find_by_id(allowed_group_id) + + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end + end + end + + def self.expire_allowed_user_ids_cache + Rails.cache.delete(ALLOWED_USER_IDS_KEY) end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b706434217d..c1ee20b6977 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,14 +19,6 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze - end - - def file_name_regex_message - "can contain only letters, digits, '_', '-', '@', '+' and '.'." - end - def container_registry_reference_regex Gitlab::PathRegex.git_reference_regex end @@ -38,8 +30,12 @@ module Gitlab @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} end + def environment_name_regex_chars + 'a-zA-Z0-9_/\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze end def environment_name_regex_message diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb deleted file mode 100644 index 36d1a8a6f64..00000000000 --- a/lib/gitlab/routes/legacy_builds.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module Routes - class LegacyBuilds - def initialize(map) - @map = map - end - - def draw - @map.instance_eval do - resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do - collection do - resources :artifacts, only: [], controller: 'build_artifacts' do - collection do - get :latest_succeeded, - path: '*ref_name_and_path', - format: false - end - end - end - - member do - get :raw - end - - resource :artifacts, only: [], controller: 'build_artifacts' do - get :download - get :browse, path: 'browse(/*path)', format: false - get :file, path: 'file/*path', format: false - get :raw, path: 'raw/*path', format: false - end - end - end - end - end - end -end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 632e2d87500..e57890f1143 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -2,10 +2,35 @@ module Gitlab module Routing extend ActiveSupport::Concern + mattr_accessor :_includers + self._includers = [] + included do + Gitlab::Routing.includes_helpers(self) + include Gitlab::Routing.url_helpers end + def self.includes_helpers(klass) + self._includers << klass + end + + def self.add_helpers(mod) + url_helpers.include mod + url_helpers.extend mod + + GitlabRoutingHelper.include mod + GitlabRoutingHelper.extend mod + + app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module + app_url_helpers.include mod + app_url_helpers.extend mod + + _includers.each do |klass| + klass.include mod + end + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0baea092e6a..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,3 +1,6 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab @@ -68,6 +71,7 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'add-project', storage, "#{name}.git"]) @@ -81,6 +85,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. @@ -99,6 +104,7 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced @@ -115,6 +121,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', storage, "#{path}.git", "#{new_path}.git"]) @@ -129,6 +136,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', forked_from_storage, "#{path}.git", forked_to_storage, @@ -143,6 +151,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'rm-project', storage, "#{name}.git"]) @@ -194,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -207,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -216,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -241,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 27696436574..e13808a2720 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -2,7 +2,7 @@ module Gitlab module SlashCommands module Presenters class Base - include Gitlab::Routing.url_helpers + include Gitlab::Routing def initialize(resource = nil) @resource = resource diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb new file mode 100644 index 00000000000..5e89e12b2b1 --- /dev/null +++ b/lib/gitlab/sql/glob.rb @@ -0,0 +1,22 @@ +module Gitlab + module SQL + module Glob + extend self + + # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern + # with SQL expression + def to_like(pattern) + <<~SQL + REPLACE(REPLACE(REPLACE(#{pattern}, + #{q('%')}, #{q('\\%')}), + #{q('_')}, #{q('\\_')}), + #{q('*')}, #{q('%')}) + SQL + end + + def q(string) + ActiveRecord::Base.connection.quote(string) + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 23af9318d1a..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Gitlab::Routing.url_helpers + include Gitlab::Routing include GitlabRoutingHelper include ActionView::RecordIdentifier @@ -23,9 +23,9 @@ module Gitlab when WikiPage wiki_page_url when ProjectSnippet - project_snippet_url(object) + project_snippet_url(object.project, object) when Snippet - personal_snippet_url(object) + snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end @@ -52,26 +52,24 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) + snippet = object.noteable if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) else - project_snippet_url(snippet, anchor: dom_id(object)) + project_snippet_url(snippet.project, snippet, anchor: dom_id(object)) end end end def wiki_page_url - namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + project_wiki_url(object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f96ee69096d..916ef365d78 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -25,27 +25,25 @@ module Gitlab RepoPath: repo_path } - if Gitlab.config.gitaly.enabled - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - - feature_enabled = case action.to_s - when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) - when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) - when 'info_refs' - true - else - raise "Unsupported action: #{action}" - end - if feature_enabled - params[:GitalyAddress] = server[:address] # This field will be deprecated - params[:GitalyServer] = server - end + server = { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + if feature_enabled + params[:GitalyAddress] = server[:address] # This field will be deprecated + params[:GitalyServer] = server end params @@ -64,10 +62,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:project_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -178,7 +187,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -194,6 +203,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb index 99f9c2c9b04..7cfe76b7b71 100644 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ b/lib/peek/rblineprof/custom_controller_helpers.rb @@ -41,9 +41,14 @@ module Peek ] end.sort_by{ |a,b,c,d,e,f| -f } - output = '' - per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| + output = "<div class='modal-dialog modal-full'><div class='modal-content'>" + output << "<div class='modal-header'>" + output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>" + output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" + output << "</div>" + output << "<div class='modal-body'>" + per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| output << "<div class='peek-rblineprof-file'><div class='heading'>" show_src = file_sort > min @@ -86,11 +91,32 @@ module Peek output << "</div></div>" # .data then .peek-rblineprof-file end - response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe + output << "</div></div></div>" + + response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe end ret end + + private + + def human_description(lineprofiler_param) + case lineprofiler_param + when 'app' + 'app/ & lib/' + when 'views' + 'app/view/' + when 'gems' + 'vendor/gems' + when 'all' + 'everything in Rails.root' + when 'stdlib' + 'everything in the Ruby standard library' + else + 'app/, config/, lib/, vendor/ & plugin/' + end + end end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d6..564aa141952 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -5,12 +5,12 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| cursor = REDIS_SCAN_START_STOP loop do cursor, keys = redis.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: REDIS_CLEAR_BATCH_SIZE ) diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e3883278886..e9fb6a008b0 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -42,8 +42,7 @@ namespace :gitlab do http_clone_url = project.http_url_to_repo ssh_clone_url = project.ssh_url_to_repo - omniauth_providers = Gitlab.config.omniauth.providers - omniauth_providers.map! { |provider| provider['name'] } + omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] } puts "" puts "GitLab information".color(:yellow) |