diff options
Diffstat (limited to 'lib')
36 files changed, 309 insertions, 393 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 29a97ccbd75..7b9de7c9598 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -16,9 +16,9 @@ module API # GET /projects/:id/access_requests get ":id/access_requests" do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - access_requesters = paginate(source.requesters.includes(:user)) + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end @@ -55,13 +55,8 @@ module API put ':id/access_requests/:user_id/approve' do required_attributes! [:user_id] source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - member = source.requesters.find_by!(user_id: params[:user_id]) - if params[:access_level] - member.update(access_level: params[:access_level]) - end - member.accept_request + member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute status :created present member.user, with: Entities::Member, member: member diff --git a/lib/api/api.rb b/lib/api/api.rb index 74ca4728695..cb47ec8f33f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -28,6 +28,7 @@ module API helpers ::SentryHelper helpers ::API::Helpers + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji mount ::API::Branches @@ -48,6 +49,7 @@ module API mount ::API::Lint mount ::API::Members mount ::API::MergeRequests + mount ::API::MergeRequestDiffs mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes @@ -70,6 +72,5 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables - mount ::API::MergeRequestDiffs end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..8cc7a26f1fa 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -33,46 +33,29 @@ module API # # If the token is revoked, then it raises RevokedError. # - # If the token is not found (nil), then it raises TokenNotFoundError. + # If the token is not found (nil), then it returns nil # # Arguments: # # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end - end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + access_token = find_access_token + return nil unless access_token + + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end @@ -96,19 +79,6 @@ module API end module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end - end - private def install_error_responders(base) diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 7c22b17e4e5..e9ccba3b465 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,23 +1,26 @@ module API class AwardEmoji < Grape::API before { authenticate! } - AWARDABLES = [Issue, MergeRequest] + AWARDABLES = %w[issue merge_request snippet] resource :projects do AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.to_s.underscore.pluralize - awardable_id_string = "#{awardable_type.to_s.underscore}_id" + awardable_string = awardable_type.pluralize + awardable_id_string = "#{awardable_type}_id" + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) @@ -27,14 +30,13 @@ module API end end - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end get "#{endpoint}/:award_id" do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji @@ -43,17 +45,14 @@ module API end end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/award_emoji + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end post endpoint do - required_attributes! [:name] - not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -65,14 +64,13 @@ module API end end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) @@ -87,9 +85,7 @@ module API helpers do def can_read_awardable? - ability = "read_#{awardable.class.to_s.underscore}".to_sym - - can?(current_user, ability, awardable) + can?(current_user, read_ability(awardable), awardable) end def can_award_awardable? @@ -100,18 +96,25 @@ module API @awardable ||= begin if params.include?(:note_id) - noteable.notes.find(params[:note_id]) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + elsif params.include?(:merge_request_id) + user_project.merge_requests.find(params[:merge_request_id]) else - noteable + user_project.snippets.find(params[:snippet_id]) end end end - def noteable - if params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) else - user_project.merge_requests.find(params[:merge_request_id]) + :"read_#{awardable.class.to_s.underscore}" end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 92a6f29adb0..04437322ec1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -15,7 +15,7 @@ module API class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :location, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end class Identity < Grape::Entity @@ -343,7 +343,7 @@ module API end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -494,6 +494,8 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :koding_enabled + expose :koding_url end class Release < Grape::Entity @@ -545,6 +547,10 @@ module API expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -552,6 +558,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -562,8 +569,8 @@ module API expose :key, :value end - class Pipeline < Grape::Entity - expose :id, :status, :ref, :sha, :before_sha, :tag, :yaml_errors + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 150875ed4f0..714d4ea3dc6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -12,13 +12,30 @@ module API nil end + def private_token + params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + def find_user_from_warden + warden ? warden.authenticate : nil + end + def find_user_by_private_token - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + token = private_token + return nil unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) end def current_user - @current_user ||= (find_user_by_private_token || doorkeeper_guard) + @current_user ||= find_user_by_private_token + @current_user ||= doorkeeper_guard + @current_user ||= find_user_from_warden unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 090d04544da..9a5d1ece070 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -90,7 +90,7 @@ module API { username: token_handler.actor_name, - lfs_token: token_handler.generate, + lfs_token: token_handler.token, repository_http_path: project.http_url_to_repo } end diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2b723b79504..767f27ef334 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -4,10 +4,9 @@ module API before { authenticate! } resource :keys do - # Get single ssh key by id. Only available to admin users. - # - # Example Request: - # GET /keys/:id + desc 'Get single ssh key by id. Only available to admin users' do + success Entities::SSHKeyWithUser + end get ":id" do authenticated_as_admin! diff --git a/lib/api/members.rb b/lib/api/members.rb index 37f0a6512f4..a18ce769e29 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -59,13 +59,6 @@ module API authorize_admin_source!(source_type, source) required_attributes! [:user_id, :access_level] - access_requester = source.requesters.find_by(user_id: params[:user_id]) - if access_requester - # We pass current_user = access_requester so that the requester doesn't - # receive a "access denied" email - ::Members::DestroyService.new(access_requester, access_requester.user).execute - end - member = source.members.find_by(user_id: params[:user_id]) # This is to ensure back-compatibility but 409 behavior should be used @@ -73,18 +66,12 @@ module API conflict!('Member already exists') if source_type == 'group' && member unless member - source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - member = source.members.find_by(user_id: params[:user_id]) + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end - if member + if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # Since `source.add_user` doesn't return a member object, we have to - # build a new one and populate its errors in order to render them. - member = source.members.build(attributes_for_keys([:user_id, :access_level, :expires_at])) - member.valid? # populate the errors - # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..9b73f6826cf 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -108,8 +108,7 @@ module API finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: @milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5eb83c2c8f8..680055c95eb 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -207,7 +207,9 @@ module API if namespace_id.present? namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) - not_found!('Target Namespace') unless namespace + unless namespace && can?(current_user, :create_projects, namespace) + not_found!('Target Namespace') + end attrs[:namespace] = namespace end @@ -391,23 +393,24 @@ module API # Share project with group # # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group + # id (required) - The ID of a project + # group_id (required) - The ID of a group # group_access (required) - Level of permissions for sharing + # expires_at (optional) - Share expiration date # # Example Request: # POST /projects/:id/share post ":id/share" do authorize! :admin_project, user_project required_attributes! [:group_id, :group_access] + attrs = attributes_for_keys [:group_id, :group_access, :expires_at] unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(attrs) + if link.save present link, with: Entities::ProjectGroupLink else diff --git a/lib/api/users.rb b/lib/api/users.rb index c440305ff0f..18c4cad09ae 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -60,6 +60,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Number of projects user can create # extern_uid - External authentication provider UID # provider - External provider @@ -74,7 +75,7 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external, :organization] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) @@ -111,6 +112,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Limit projects each user can create # bio - Bio # location - Location of the user @@ -122,7 +124,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external, :organization] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 16cd774c81a..affe34394c2 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -64,7 +64,7 @@ module Banzai end end - def project_from_ref_cache(ref) + def project_from_ref_cached(ref) if RequestStore.active? cache = project_refs_cache @@ -146,7 +146,7 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_text: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cache(project_ref) + project = project_from_ref_cached(project_ref) if project && object = find_object_cached(project, id) title = object_link_title(object) @@ -243,11 +243,27 @@ module Banzai end end - # Returns the projects for the given paths. - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) Project.where_paths_in(paths).includes(:namespace) end + # Returns projects for the given paths. + def find_projects_for_paths(paths) + if RequestStore.active? + to_query = paths - project_refs_cache.keys + + unless to_query.empty? + projects_relation_for_paths(to_query).each do |project| + get_or_set_cache(project_refs_cache, project.path_with_namespace) { project } + end + end + + project_refs_cache.slice(*paths).values + else + projects_relation_for_paths(paths) + end + end + def current_project_path @current_project_path ||= project.path_with_namespace end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4042e9a4c25..54c5f9a71a4 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -66,7 +66,7 @@ module Banzai end end - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6e13282d5f4..2470362e019 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -7,7 +7,7 @@ module Banzai UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze def whitelist - whitelist = super.dup + whitelist = super customize_whitelist(whitelist) @@ -42,58 +42,58 @@ module Banzai # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') - whitelist[:transformers] = whitelist[:transformers].dup - # ...but then remove links with unsafe protocols - whitelist[:transformers].push(remove_unsafe_links) + whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(remove_rel) + whitelist[:transformers].push(self.class.remove_rel) # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(clean_spans) + whitelist[:transformers].push(self.class.clean_spans) whitelist end - def remove_unsafe_links - lambda do |env| - node = env[:node] + class << self + def remove_unsafe_links + lambda do |env| + node = env[:node] - return unless node.name == 'a' - return unless node.has_attribute?('href') + return unless node.name == 'a' + return unless node.has_attribute?('href') - begin - uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + begin + uri = Addressable::URI.parse(node['href']) + uri.scheme = uri.scheme.strip.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + rescue Addressable::URI::InvalidURIError + node.remove_attribute('href') + end end end - end - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end end end - end - def clean_spans - lambda do |env| - node = env[:node] + def clean_spans + lambda do |env| + node = env[:node] - return unless node.name == 'span' - return unless node.has_attribute?('class') + return unless node.name == 'span' + return unless node.has_attribute?('class') - unless has_ancestor?(node, 'pre') - node.remove_attribute('class') - end + unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } + node.remove_attribute('class') + end - { node_whitelist: [node] } + { node_whitelist: [node] } + end end end end diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..9fa5f589f3e 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,27 +2,7 @@ require 'task_list/filter' module Banzai module Filter - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) - if new_class_names.include?('task-list') - # Don't add class to all lists - return - elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') - end - - add_css_class_without_fix(node, *new_class_names) - end - - alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index e8e03e4a98f..f5d110e987b 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -79,7 +79,11 @@ module Banzai def referenced_by(nodes) ids = unique_attribute_values(nodes, self.class.data_attribute) - references_relation.where(id: ids) + if ids.empty? + references_relation.none + else + references_relation.where(id: ids) + end end # Returns the ActiveRecord::Relation to use for querying references in the diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 0369e80312a..2fd1fced65c 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,7 +4,7 @@ module Ci include Gitlab::Ci::Config::Node::LegacyValidationHelpers - attr_reader :path, :cache, :stages + attr_reader :path, :cache, :stages, :jobs def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb index 3da04edde70..997377abc55 100644 --- a/lib/ci/mask_secret.rb +++ b/lib/ci/mask_secret.rb @@ -1,9 +1,10 @@ module Ci::MaskSecret class << self - def mask(value, token) + def mask!(value, token) return value unless value.present? && token.present? - value.gsub(token, 'x' * token.length) + value.gsub!(token, 'x' * token.length) + value end end end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index a533bac2692..9b484a2ecfd 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -53,6 +53,10 @@ module Gitlab } end + def sym_options_with_owner + sym_options.merge(owner: OWNER) + end + def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7c0f2115d43..aca5d0020cf 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -124,7 +124,7 @@ module Gitlab read_authentication_abilities end - Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.value, password) + Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password) end def build_access_token_check(login, password) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 12fbb78c53e..ef9160d6437 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -59,10 +59,8 @@ module Gitlab # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false - ENV['USE_DB'] != 'false' && active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 3ab99360206..3cd515e4a3a 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -19,6 +19,8 @@ module Gitlab end def committer_hash(email:, name:) + return if email.nil? || name.nil? + { email: email, name: name, diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 084e514492c..e33ac61f5ae 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -52,7 +52,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - request { api.send(method, *args, &block) } + request(method, *args, &block) else super(method, *args, &block) end @@ -99,20 +99,19 @@ module Gitlab rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME end - def request + def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = yield + data = api.send(method, *args, &block) + yield data last_response = api.last_response while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get - data.concat(last_response.data) if last_response.data.is_a?(Array) + yield last_response.data if last_response.data.is_a?(Array) end - - data end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index d35ee2a1c65..b8321244473 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -10,6 +10,7 @@ module Gitlab @repo = project.import_source @repo_url = project.import_url @errors = [] + @labels = {} if credentials @client = Client.new(credentials[:user]) @@ -23,6 +24,7 @@ module Gitlab import_milestones import_issues import_pull_requests + import_comments import_wiki import_releases handle_errors @@ -46,66 +48,68 @@ module Gitlab end def import_labels - labels = client.labels(repo, per_page: 100) - - labels.each do |raw| - begin - LabelFormatter.new(project, raw).create! - rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + client.labels(repo, per_page: 100) do |labels| + labels.each do |raw| + begin + label = LabelFormatter.new(project, raw).create! + @labels[label.title] = label.id + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end end def import_milestones - milestones = client.milestones(repo, state: :all, per_page: 100) - - milestones.each do |raw| - begin - MilestoneFormatter.new(project, raw).create! - rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + client.milestones(repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + MilestoneFormatter.new(project, raw).create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end end def import_issues - issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - - issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) - - if gh_issue.valid? - begin - issue = gh_issue.create! - apply_labels(issue) - import_comments(issue) if gh_issue.has_comments? - rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw) + + if gh_issue.valid? + begin + issue = gh_issue.create! + apply_labels(issue, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end end end def import_pull_requests - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) - - pull_requests.each do |pull_request| - begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? - - merge_request = pull_request.create! - apply_labels(merge_request) - import_comments(merge_request) - import_comments_on_diff(merge_request) - rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } - ensure - clean_up_restored_branches(pull_request) + client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + pull_requests.each do |raw| + pull_request = PullRequestFormatter.new(project, raw) + next unless pull_request.valid? + + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request = pull_request.create! + apply_labels(merge_request, raw) + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(pull_request) + end end end + + project.repository.after_remove_branch end def restore_source_branch(pull_request) @@ -125,37 +129,38 @@ module Gitlab def clean_up_restored_branches(pull_request) remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? - - project.repository.after_remove_branch end - def apply_labels(issuable) - issue = client.issue(repo, issuable.iid) - - if issue.labels.count > 0 - label_ids = issue.labels - .map { |attrs| project.labels.find_by(title: attrs.name).try(:id) } + def apply_labels(issuable, raw_issuable) + if raw_issuable.labels.count > 0 + label_ids = raw_issuable.labels + .map { |attrs| @labels[attrs.name] } .compact issuable.update_attribute(:label_ids, label_ids) end end - def import_comments(issuable) - comments = client.issue_comments(repo, issuable.iid, per_page: 100) - create_comments(issuable, comments) - end + def import_comments + client.issues_comments(repo, per_page: 100) do |comments| + create_comments(comments, :issue) + end - def import_comments_on_diff(merge_request) - comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) - create_comments(merge_request, comments) + client.pull_requests_comments(repo, per_page: 100) do |comments| + create_comments(comments, :pull_request) + end end - def create_comments(issuable, comments) + def create_comments(comments, issuable_type) ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw) + issuable_class = issuable_type == :issue ? Issue : MergeRequest + iid = raw.send("#{issuable_type}_url").split('/').last # GH doesn't return parent ID directly + issuable = issuable_class.find_by_iid(iid) + next unless issuable + issuable.notes.create!(comment.attributes) rescue => e errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } @@ -180,13 +185,14 @@ module Gitlab end def import_releases - releases = client.releases(repo, per_page: 100) - releases.each do |raw| - begin - gh_release = ReleaseFormatter.new(project, raw) - gh_release.create! if gh_release.valid? - rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + client.releases(repo, per_page: 100) do |releases| + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c5a11148d33..2c21804fe7a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,7 +11,6 @@ module Gitlab if current_user gon.current_user_id = current_user.id - gon.api_token = current_user.private_token end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 925a952156f..bb9d1080330 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,5 +1,8 @@ # Model relationships to be included in the project import/export project_tree: + - :labels + - milestones: + - :events - issues: - :events - notes: @@ -10,6 +13,7 @@ project_tree: - milestone: - :events - snippets: + - :award_emoji - notes: :author - :releases @@ -38,9 +42,6 @@ project_tree: - protected_branches: - :merge_access_levels - :push_access_levels - - :labels - - milestones: - - :events - :project_feature # Only include the following attributes for the models specified. @@ -66,9 +67,13 @@ excluded_attributes: - :milestone_id merge_requests: - :milestone_id + award_emoji: + - :awardable_id methods: statuses: - :type + services: + - :type merge_request_diff: - :utf8_st_diffs diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7b3551b84c..35ff134ea19 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -61,11 +61,17 @@ module Gitlab def restore_project return @project unless @tree_hash - project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } @project.update(project_params) @project end + def project_params + @tree_hash.reject do |key, value| + # return params that are not 1 to many or 1 to 1 relations + value.is_a?(Array) || key == key.singularize + end + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 2f326d00a2f..7e06bd2b0fb 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -51,8 +51,6 @@ module Gitlab user.ldap_block false end - rescue - false end def adapter diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 82cb8cef754..8b38cfaefb6 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -62,6 +62,9 @@ module Gitlab results end end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] rescue Timeout::Error Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") [] diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index d089a2f9b0b..5f67e97fa2a 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -17,19 +17,13 @@ module Gitlab end end - def generate - token = Devise.friendly_token(TOKEN_LENGTH) - + def token Gitlab::Redis.with do |redis| + token = redis.get(redis_key) + token ||= Devise.friendly_token(TOKEN_LENGTH) redis.set(redis_key, token, ex: EXPIRY_TIME) - end - token - end - - def value - Gitlab::Redis.with do |redis| - redis.get(redis_key) + token end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9376b54f43b..3faab937726 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,35 +9,35 @@ module Gitlab SIDEKIQ_NAMESPACE = 'resque:gitlab' MAILROOM_NAMESPACE = 'mail_room:gitlab' DEFAULT_REDIS_URL = 'redis://localhost:6379' - - # To be thread-safe we must be careful when writing the class instance - # variables @url and @pool. Because @pool depends on @url we need two - # mutexes to prevent deadlock. - PARAMS_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :PARAMS_MUTEX, :POOL_MUTEX + CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) class << self + # Do NOT cache in an instance variable. Result may be mutated by caller. def params - @params || PARAMS_MUTEX.synchronize { @params = new.params } + new.params end + # Do NOT cache in an instance variable. Result may be mutated by caller. # @deprecated Use .params instead to get sentinel support def url new.url end def with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(params) } - end - end + @pool ||= ConnectionPool.new { ::Redis.new(params) } @pool.with { |redis| yield redis } end - def reset_params! - @params = nil + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = File.read(CONFIG_FILE).freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config end end @@ -83,12 +83,7 @@ module Gitlab end def fetch_config - file = config_file - File.exist?(file) ? YAML.load_file(file)[@rails_env] : false - end - - def config_file - File.expand_path('../../../config/resque.yml', __FILE__) + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end end end diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 7813091ec7b..82a59a7a87e 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -2,7 +2,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{job['args']}" + Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" yield end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 60aae541d46..5d33f98e89e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -60,7 +60,7 @@ module Gitlab def send_git_diff(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -73,7 +73,7 @@ module Gitlab def send_git_patch(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -107,15 +107,15 @@ module Gitlab bytes end end - + def write_secret bytes = SecureRandom.random_bytes(SECRET_LENGTH) - File.open(secret_path, 'w:BINARY', 0600) do |f| + File.open(secret_path, 'w:BINARY', 0600) do |f| f.chmod(0600) f.write(Base64.strict_encode64(bytes)) end end - + def verify_api_request!(request_headers) JWT.decode( request_headers[INTERNAL_API_REQUEST_HEADER], @@ -128,7 +128,7 @@ module Gitlab def secret_path Rails.root.join('.gitlab_workhorse_secret') end - + protected def encode(hash) diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end |