diff options
Diffstat (limited to 'lib')
154 files changed, 4318 insertions, 1214 deletions
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c4c0fdda665..87b9db66efd 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,6 +42,38 @@ module API # Helper Methods for Grape Endpoint module HelperMethods + def find_current_user + user = + find_user_from_private_token || + find_user_from_oauth_token || + find_user_from_warden + + return nil unless user + + raise UnauthorizedError unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + + user + end + + def private_token + params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + end + + private + + def find_user_from_private_token + token_string = private_token.to_s + return nil unless token_string.present? + + user = + find_user_by_authentication_token(token_string) || + find_user_by_personal_access_token(token_string) + + raise UnauthorizedError unless user + + user + end + # Invokes the doorkeeper guard. # # If token is presented and valid, then it sets @current_user. @@ -60,59 +92,89 @@ module API # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard(scopes: []) - access_token = find_access_token - return nil unless access_token + def find_user_from_oauth_token + access_token = find_oauth_access_token + return unless access_token - case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) - when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + find_user_by_access_token(access_token) + end - when AccessTokenValidationService::EXPIRED - raise ExpiredError + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end - when AccessTokenValidationService::REVOKED - raise RevokedError + def find_user_by_personal_access_token(token_string) + access_token = PersonalAccessToken.find_by_token(token_string) + return unless access_token - when AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + find_user_by_access_token(access_token) end - def find_user_by_private_token(scopes: []) - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - - return nil unless token_string.present? + # Check the Rails session for valid authentication details + def find_user_from_warden + warden.try(:authenticate) if verified_request? + end - find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + def warden + env['warden'] end - def current_user - @current_user + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(env) end - private + def find_oauth_access_token + return @oauth_access_token if defined?(@oauth_access_token) - def find_user_by_authentication_token(token_string) - User.find_by_authentication_token(token_string) - end + token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + return @oauth_access_token = nil unless token - def find_user_by_personal_access_token(token_string, scopes) - access_token = PersonalAccessToken.active.find_by_token(token_string) - return unless access_token + @oauth_access_token = OauthAccessToken.by_token(token) + raise UnauthorizedError unless @oauth_access_token - if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes) - User.find(access_token.user_id) - end + @oauth_access_token.revoke_previous_refresh_token! + @oauth_access_token end - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + def find_user_by_access_token(access_token) + scopes = scopes_registered_for_endpoint + + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + + when AccessTokenValidationService::EXPIRED + raise ExpiredError + + when AccessTokenValidationService::REVOKED + raise RevokedError + + when AccessTokenValidationService::VALID + access_token.user + end end def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end + + # An array of scopes that were registered (using `allow_access_with_scope`) + # for the current endpoint class. It also returns scopes registered on + # `API::API`, since these are meant to apply to all API routes. + def scopes_registered_for_endpoint + @scopes_registered_for_endpoint ||= + begin + endpoint_classes = [options[:for].presence, ::API::API].compact + endpoint_classes.reduce([]) do |memo, endpoint| + if endpoint.respond_to?(:allowed_scopes) + memo.concat(endpoint.allowed_scopes) + else + memo + end + end + end + end end module ClassMethods @@ -169,6 +231,7 @@ module API TokenNotFoundError = Class.new(StandardError) ExpiredError = Class.new(StandardError) RevokedError = Class.new(StandardError) + UnauthorizedError = Class.new(StandardError) class InsufficientScopeError < StandardError attr_reader :scopes diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 643c8e6fb8e..61a2d688282 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -13,7 +13,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository branches' do - success Entities::RepoBranch + success Entities::Branch end params do use :pagination @@ -23,13 +23,13 @@ module API # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 Gitlab::GitalyClient.allow_n_plus_1_calls do - present paginate(branches), with: Entities::RepoBranch, project: user_project + present paginate(branches), with: Entities::Branch, project: user_project end end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do desc 'Get a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -41,7 +41,7 @@ module API branch = user_project.repository.find_branch(params[:branch]) not_found!('Branch') unless branch - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end end @@ -50,7 +50,7 @@ module API # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. desc 'Protect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -80,7 +80,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -88,7 +88,7 @@ module API # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -101,11 +101,11 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end desc 'Create branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -119,7 +119,7 @@ module API if result[:status] == :success present result[:branch], - with: Entities::RepoBranch, + with: Entities::Branch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 67a5264adc8..2685dc27252 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -11,7 +11,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do - success Entities::RepoCommit + success Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -44,11 +44,11 @@ module API paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present paginate(paginated_commits), with: Entities::RepoCommit + present paginate(paginated_commits), with: Entities::Commit end desc 'Commit multiple file changes as one commit' do - success Entities::RepoCommitDetail + success Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -70,14 +70,14 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - present commit_detail, with: Entities::RepoCommitDetail + present commit_detail, with: Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success Entities::RepoCommitDetail + success Entities::CommitDetail failure [[404, 'Commit Not Found']] end params do @@ -88,7 +88,7 @@ module API not_found! 'Commit' unless commit - present commit, with: Entities::RepoCommitDetail + present commit, with: Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -102,7 +102,7 @@ module API not_found! 'Commit' unless commit - present commit.raw_diffs.to_a, with: Entities::RepoDiff + present commit.raw_diffs.to_a, with: Entities::Diff end desc "Get a commit's comments" do @@ -124,7 +124,7 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success Entities::RepoCommit + success Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' @@ -149,7 +149,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit else render_api_error!(result[:message], 400) end @@ -184,7 +184,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb new file mode 100644 index 00000000000..5000aa0d9ac --- /dev/null +++ b/lib/api/custom_attributes_endpoints.rb @@ -0,0 +1,77 @@ +module API + module CustomAttributesEndpoints + extend ActiveSupport::Concern + + included do + attributable_class = name.demodulize.singularize + attributable_key = attributable_class.underscore + attributable_name = attributable_class.humanize(capitalize: false) + attributable_finder = "find_#{attributable_key}" + + helpers do + params :custom_attributes_key do + requires :key, type: String, desc: 'The key of the custom attribute' + end + end + + desc "Get all custom attributes on a #{attributable_name}" do + success Entities::CustomAttribute + end + get ':id/custom_attributes' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + present resource.custom_attributes, with: Entities::CustomAttribute + end + + desc "Get a custom attribute on a #{attributable_name}" do + success Entities::CustomAttribute + end + params do + use :custom_attributes_key + end + get ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + custom_attribute = resource.custom_attributes.find_by!(key: params[:key]) + + present custom_attribute, with: Entities::CustomAttribute + end + + desc "Set a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + requires :value, type: String, desc: 'The value of the custom attribute' + end + put ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + custom_attribute = resource.custom_attributes + .find_or_initialize_by(key: params[:key]) + + custom_attribute.update(value: params[:value]) + + if custom_attribute.valid? + present custom_attribute, with: Entities::CustomAttribute + else + render_validation_error!(custom_attribute) + end + end + + desc "Delete a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + end + delete ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + resource.custom_attributes.find_by!(key: params[:key]).destroy + + status 204 + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 71253f72533..5f0bad14839 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -89,6 +89,9 @@ module API expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :avatar_url do |project, options| + project.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :created_at, :last_activity_at end @@ -146,9 +149,7 @@ module API expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs @@ -193,8 +194,8 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) + expose :avatar_url do |group, options| + group.avatar_url(only_path: false) end expose :web_url expose :request_access_enabled @@ -219,7 +220,7 @@ module API expose :shared_projects, using: Entities::Project end - class RepoCommit < Grape::Entity + class Commit < Grape::Entity expose :id, :short_id, :title, :created_at expose :parent_ids expose :safe_message, as: :message @@ -227,19 +228,20 @@ module API expose :committer_name, :committer_email, :committed_date end - class RepoCommitStats < Grape::Entity + class CommitStats < Grape::Entity expose :additions, :deletions, :total end - class RepoCommitDetail < RepoCommit - expose :stats, using: Entities::RepoCommitStats + class CommitDetail < Commit + expose :stats, using: Entities::CommitStats expose :status + expose :last_pipeline, using: 'API::Entities::PipelineBasic' end - class RepoBranch < Grape::Entity + class Branch < Grape::Entity expose :name - expose :commit, using: Entities::RepoCommit do |repo_branch, options| + expose :commit, using: Entities::Commit do |repo_branch, options| options[:project].repository.commit(repo_branch.dereferenced_target) end @@ -263,7 +265,7 @@ module API end end - class RepoTreeObject < Grape::Entity + class TreeObject < Grape::Entity expose :id, :name, :type, :path expose :mode do |obj, options| @@ -303,7 +305,7 @@ module API expose :state, :created_at, :updated_at end - class RepoDiff < Grape::Entity + class Diff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file @@ -366,6 +368,7 @@ module API end expose :due_date expose :confidential + expose :discussion_locked expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) @@ -462,6 +465,7 @@ module API expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :user_notes_count + expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -481,7 +485,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -492,9 +496,9 @@ module API end class MergeRequestDiffFull < MergeRequestDiff - expose :commits, using: Entities::RepoCommit + expose :commits, using: Entities::Commit - expose :diffs, using: Entities::RepoDiff do |compare, _| + expose :diffs, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -590,8 +594,7 @@ module API expose :target_type expose :target do |todo, options| - target = todo.target_type == 'Commit' ? 'RepoCommit' : todo.target_type - Entities.const_get(target).represent(todo.target, options) + Entities.const_get(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| @@ -727,15 +730,15 @@ module API end class Compare < Grape::Entity - expose :commit, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil).last + expose :commit, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil).last end - expose :commits, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil) + expose :commits, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil) end - expose :diffs, using: Entities::RepoDiff do |compare, options| + expose :diffs, using: Entities::Diff do |compare, options| compare.diffs(limits: false).to_a end @@ -771,10 +774,10 @@ module API expose :description end - class RepoTag < Grape::Entity + class Tag < Grape::Entity expose :name, :message - expose :commit, using: Entities::RepoCommit do |repo_tag, options| + expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) end @@ -825,7 +828,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: User expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } - expose :commit, with: RepoCommit + expose :commit, with: Commit expose :runner, with: Runner expose :pipeline, with: PipelineBasic end @@ -878,7 +881,7 @@ module API expose :deployable, using: Entities::Job end - class RepoLicense < Grape::Entity + class License < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular expose :url, as: :html_url @@ -1020,6 +1023,7 @@ module API expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency + expose :features end end @@ -1034,5 +1038,10 @@ module API expose :failing_on_hosts expose :total_failures end + + class CustomAttribute < Grape::Entity + expose :key + expose :value + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 00dbc2aee7a..2b316b58ed9 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -139,7 +139,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(initial_current_user, :access_api) + unauthorized! unless current_user end def authenticate_non_get! @@ -285,7 +285,7 @@ module API if sentry_enabled? && report_exception?(exception) define_params_for_grape_middleware sentry_context - Raven.capture_exception(exception) + Raven.capture_exception(exception, extra: params) end # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 @@ -377,36 +377,13 @@ module API private - def private_token - params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] - end - - def warden - env['warden'] - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(env) - end - - # Check the Rails session for valid authentication details - def find_user_from_warden - warden.try(:authenticate) if verified_request? - end - def initial_current_user return @initial_current_user if defined?(@initial_current_user) - Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= find_user_from_warden - - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil - end - @initial_current_user + begin + @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user } + rescue APIGuard::UnauthorizedError + unauthorized! end end @@ -454,10 +431,12 @@ module API header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) end - # The Grape Error Middleware only has access to env but no params. We workaround this by - # defining a method that returns the right value. + # The Grape Error Middleware only has access to `env` but not `params` nor + # `request`. We workaround this by defining methods that returns the right + # values. def define_params_for_grape_middleware - self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:params) { request.params.symbolize_keys } end # We could get a Grape or a standard Ruby exception. We should only report anything that @@ -467,22 +446,5 @@ module API exception.status == 500 end - - # An array of scopes that were registered (using `allow_access_with_scope`) - # for the current endpoint class. It also returns scopes registered on - # `API::API`, since these are meant to apply to all API routes. - def scopes_registered_for_endpoint - @scopes_registered_for_endpoint ||= - begin - endpoint_classes = [options[:for].presence, ::API::API].compact - endpoint_classes.reduce([]) do |memo, endpoint| - if endpoint.respond_to?(:allowed_scopes) - memo.concat(endpoint.allowed_scopes) - else - memo - end - end - end - end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index c0fef56378f..6e78ac2c903 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -31,6 +31,12 @@ module API protocol = params[:protocol] actor.update_last_used_at if actor.is_a?(Key) + user = + if actor.is_a?(Key) + actor.user + else + actor + end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass @@ -47,6 +53,7 @@ module API { status: true, gl_repository: gl_repository, + gl_username: user&.username, repository_path: repository_path, gitaly: gitaly_payload(params[:action]) } @@ -136,7 +143,7 @@ module API codes = nil - ::Users::UpdateService.new(user).execute! do |user| + ::Users::UpdateService.new(current_user, user: user).execute! do |user| codes = user.generate_otp_backup_codes! end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1729df2aad0..0df41dcc903 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -48,6 +48,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" end params :issue_params do @@ -193,7 +194,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8aa1e0216ee..be843ec8251 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -183,13 +183,13 @@ module API end desc 'Get the commits of a merge request' do - success Entities::RepoCommit + success Entities::Commit end get ':id/merge_requests/:merge_request_iid/commits' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = ::Kaminari.paginate_array(merge_request.commits) - present paginate(commits), with: Entities::RepoCommit + present paginate(commits), with: Entities::Commit end desc 'Show the merge request changes' do @@ -214,12 +214,14 @@ module API :remove_source_branch, :state_event, :target_branch, - :title + :title, + :discussion_locked ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], desc: 'Status of the merge request' + optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' use :optional_params at_least_one_of(*at_least_one_of_ce) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d6e7203adaf..0b9ab4eeb05 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -78,6 +78,8 @@ module API } if can?(current_user, noteable_read_ability_name(noteable), noteable) + authorize! :create_note, noteable + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bcc0833aa5c..0266bf2f717 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -35,7 +35,7 @@ module API new_notification_email = params.delete(:notification_email) if new_notification_email - ::Users::UpdateService.new(current_user, notification_email: new_notification_email).execute + ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute end notification_setting.update(declared_params(include_missing: false)) diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 70f7bf32a71..7887b886c03 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -35,7 +35,7 @@ module API end desc 'Get a project repository tree' do - success Entities::RepoTreeObject + success Entities::TreeObject end params do optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -52,7 +52,7 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) entries = ::Kaminari.paginate_array(tree.sorted_entries) - present paginate(entries), with: Entities::RepoTreeObject + present paginate(entries), with: Entities::TreeObject end desc 'Get raw blob contents from the repository' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 912415e3a7f..0d394a7b441 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -11,18 +11,18 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository tags' do - success Entities::RepoTag + success Entities::Tag end params do use :pagination end get ':id/repository/tags' do tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) - present paginate(tags), with: Entities::RepoTag, project: user_project + present paginate(tags), with: Entities::Tag, project: user_project end desc 'Get a single repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -31,11 +31,11 @@ module API tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag - present tag, with: Entities::RepoTag, project: user_project + present tag, with: Entities::Tag, project: user_project end desc 'Create a new repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -51,7 +51,7 @@ module API if result[:status] == :success present result[:tag], - with: Entities::RepoTag, + with: Entities::Tag, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index f70bc0622b7..6550b331fb8 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -49,7 +49,7 @@ module API desc 'Get the list of the available license template' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -60,12 +60,12 @@ module API featured: declared(params)[:popular].present? ? true : nil } licences = ::Kaminari.paginate_array(Licensee::License.all(options)) - present paginate(licences), with: Entities::RepoLicense + present paginate(licences), with: Entities::License end desc 'Get the text for a specific license' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -75,7 +75,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| diff --git a/lib/api/users.rb b/lib/api/users.rb index bdebda58d3f..b6f97a1eac2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,12 +6,14 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + include CustomAttributesEndpoints + before do authenticate_non_get! end helpers do - def find_user(params) + def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end @@ -166,7 +168,7 @@ module API user_params[:password_expires_at] = Time.now if user_params[:password].present? - result = ::Users::UpdateService.new(user, user_params.except(:extern_uid, :provider)).execute + result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute if result[:status] == :success present user, with: Entities::UserPublic @@ -326,10 +328,9 @@ module API user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - email = Emails::CreateService.new(user, declared_params(include_missing: false)).execute + email = Emails::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -367,10 +368,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: user).execute(email) end - - user.update_secondary_emails! end desc 'Delete a user. Available only for admins.' do @@ -430,7 +429,7 @@ module API resource :impersonation_tokens do helpers do def finder(options = {}) - user = find_user(params) + user = find_user_by_id(params) PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end @@ -672,10 +671,9 @@ module API requires :email, type: String, desc: 'The new email' end post "emails" do - email = Emails::CreateService.new(current_user, declared_params).execute + email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -691,10 +689,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: current_user).execute(email) end - - current_user.update_secondary_emails! end desc 'Get a list of user activities' diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index 81b13249892..69cd12de72c 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -11,12 +11,12 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository branches' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) - present branches, with: ::API::Entities::RepoBranch, project: user_project + present branches, with: ::API::Entities::Branch, project: user_project end desc 'Delete a branch' @@ -47,7 +47,7 @@ module API end desc 'Create branch' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end params do requires :branch_name, type: String, desc: 'The name of the branch' @@ -60,7 +60,7 @@ module API if result[:status] == :success present result[:branch], - with: ::API::Entities::RepoBranch, + with: ::API::Entities::Branch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 759bb998c14..ed206a6def0 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -13,7 +13,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -34,11 +34,11 @@ module API after: params[:since], before: params[:until]) - present commits, with: ::API::Entities::RepoCommit + present commits, with: ::API::Entities::Commit end desc 'Commit multiple file changes as one commit' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -59,14 +59,14 @@ module API if result[:status] == :success commit_detail = user_project.repository.commits(result[:result], limit: 1).first - present commit_detail, with: ::API::Entities::RepoCommitDetail + present commit_detail, with: ::API::Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail failure [[404, 'Not Found']] end params do @@ -77,7 +77,7 @@ module API not_found! "Commit" unless commit - present commit, with: ::API::Entities::RepoCommitDetail + present commit, with: ::API::Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -113,7 +113,7 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha to be cherry picked' @@ -138,7 +138,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::Commit else render_api_error!(result[:message], 400) end @@ -173,7 +173,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c928ce5265b..afdd7b83998 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -220,7 +220,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: ::API::Entities::User expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: ::API::Entities::RepoCommit + expose :commit, with: ::API::Entities::Commit expose :runner, with: ::API::Entities::Runner expose :pipeline, with: ::API::Entities::PipelineBasic end @@ -237,7 +237,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: ::API::Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index b6b7254ae29..1d6d823f32b 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -135,12 +135,12 @@ module API end desc 'Get the commits of a merge request' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end get "#{path}/commits" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.commits, with: ::API::Entities::RepoCommit + present merge_request.commits, with: ::API::Entities::Commit end desc 'Show the merge request changes' do diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index b9bc9a99093..f9a47101e27 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -19,7 +19,7 @@ module API end desc 'Get a project repository tree' do - success ::API::Entities::RepoTreeObject + success ::API::Entities::TreeObject end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -35,7 +35,7 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - present tree.sorted_entries, with: ::API::Entities::RepoTreeObject + present tree.sorted_entries, with: ::API::Entities::TreeObject end desc 'Get a raw file contents' diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb index 7e5875cd030..6e37d31d153 100644 --- a/lib/api/v3/tags.rb +++ b/lib/api/v3/tags.rb @@ -8,11 +8,11 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository tags' do - success ::API::Entities::RepoTag + success ::API::Entities::Tag end get ":id/repository/tags" do tags = user_project.repository.tags.sort_by(&:name).reverse - present tags, with: ::API::Entities::RepoTag, project: user_project + present tags, with: ::API::Entities::Tag, project: user_project end desc 'Delete a repository tag' diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 2a2fb59045c..7298203df10 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -52,7 +52,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -61,7 +61,7 @@ module API options = { featured: declared(params)[:popular].present? ? true : nil } - present Licensee::License.all(options), with: ::API::Entities::RepoLicense + present Licensee::License.all(options), with: ::API::Entities::License end end @@ -70,7 +70,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -80,7 +80,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4e92be85110..3ad09a1b421 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -78,7 +78,7 @@ module Backup project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) else %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ee73fa91589..9cac303e645 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,6 +1,18 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + REDCARPET_OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + def initialize(text, context = nil, result = nil) super text, context, result @text = @text.delete "\r" @@ -13,27 +25,11 @@ module Banzai end def self.renderer - @renderer ||= begin + Thread.current[:banzai_markdown_renderer] ||= begin renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) end end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - private_class_method :redcarpet_options end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 9923ec4e870..d8c8deea628 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -45,8 +45,9 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) - # Disallow `name` attribute globally + # Disallow `name` attribute globally, allow on `a` whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') @@ -72,8 +73,9 @@ module Banzai return unless node.has_attribute?('href') begin + node['href'] = node['href'].strip uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + uri.scheme = uri.scheme.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ceca9296851..5f91884a878 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -40,7 +40,7 @@ module Banzai return cacheless_render_field(object, field) end - object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) + object.refresh_markdown_cache! unless object.cached_html_up_to_date?(field) object.cached_html_for(field) end @@ -162,10 +162,5 @@ module Banzai return unless cache_key Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end - - # GitLab EE needs to disable updates on GET requests in Geo - def self.update_object?(object) - true - end end end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index bfcec241489..7cfa82a9a9f 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -206,11 +206,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return false if passes.any? { |p| p == false } - return true if passes.all? { |p| p == true } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == false + end + + true end def repr @@ -245,11 +247,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return true if passes.any? { |p| p == true } - return false if passes.all? { |p| p == false } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == true + end + + false end def score(context) diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 56afd1f1392..45ff2ef9ced 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -107,7 +107,7 @@ module DeclarativePolicy end # This is the core spot where all those `#score` methods matter. - # It is critcal for performance to run steps in the correct order, + # It is critical for performance to run steps in the correct order, # so that we don't compute expensive conditions (potentially n times # if we're called on, say, a large list of users). # @@ -139,30 +139,39 @@ module DeclarativePolicy return end - steps = Set.new(@steps) - remaining_enablers = steps.count { |s| s.enable? } + remaining_steps = Set.new(@steps) + remaining_enablers, remaining_preventers = remaining_steps.partition(&:enable?).map { |s| Set.new(s) } loop do - return if steps.empty? + if @state.enabled? + # Once we set this, we never need to unset it, because a single + # prevent will stop this from being enabled + remaining_steps = remaining_preventers + else + # if the permission hasn't yet been enabled and we only have + # prevent steps left, we short-circuit the state here + @state.prevent! if remaining_enablers.empty? + end - # if the permission hasn't yet been enabled and we only have - # prevent steps left, we short-circuit the state here - @state.prevent! if !@state.enabled? && remaining_enablers == 0 + return if remaining_steps.empty? lowest_score = Float::INFINITY next_step = nil - steps.each do |step| + remaining_steps.each do |step| score = step.score + if score < lowest_score next_step = step lowest_score = score end - end - steps.delete(next_step) + break if lowest_score.zero? + end - remaining_enablers -= 1 if next_step.enable? + [remaining_steps, remaining_enablers, remaining_preventers].each do |set| + set.delete(next_step) + end yield next_step, lowest_score end diff --git a/lib/github/client.rb b/lib/github/client.rb index 9c476df7d46..29bd9c1f39e 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,6 +1,7 @@ module Github class Client TIMEOUT = 60 + DEFAULT_PER_PAGE = 100 attr_reader :connection, :rate_limit @@ -20,7 +21,7 @@ module Github exceed, reset_in = rate_limit.get sleep reset_in if exceed - Github::Response.new(connection.get(url, query)) + Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) end private diff --git a/lib/github/import.rb b/lib/github/import.rb index 9354e142d3d..55f8387f27a 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,48 +1,15 @@ require_relative 'error' +require_relative 'import/issue' +require_relative 'import/legacy_diff_note' +require_relative 'import/merge_request' +require_relative 'import/note' module Github class Import include Gitlab::ShellAdapter - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Note < ::Note - self.table_name = 'notes' - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose + :options, :errors, :cached, :verbose, :last_fetched_at def initialize(project, options = {}) @project = project @@ -54,12 +21,13 @@ module Github @verbose = options.fetch(:verbose, false) @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] + @last_fetched_at = nil end # rubocop: disable Rails/Output def execute puts 'Fetching repository...'.color(:aqua) if verbose - fetch_repository + setup_and_fetch_repository puts 'Fetching labels...'.color(:aqua) if verbose fetch_labels puts 'Fetching milestones...'.color(:aqua) if verbose @@ -75,7 +43,7 @@ module Github puts 'Expiring repository cache...'.color(:aqua) if verbose expire_repository_cache - true + errors.empty? rescue Github::RepositoryFetchError expire_repository_cache false @@ -85,18 +53,24 @@ module Github private - def fetch_repository + def setup_and_fetch_repository begin project.ensure_repository project.repository.add_remote('github', repo_url) - project.repository.set_remote_as_mirror('github') - project.repository.fetch_remote('github', forced: true) + project.repository.set_import_remote_as_mirror('github') + project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') + fetch_remote(forced: true) rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e error(:project, repo_url, e.message) raise Github::RepositoryFetchError end end + def fetch_remote(forced: false) + @last_fetched_at = Time.now + project.repository.fetch_remote('github', forced: forced) + end + def fetch_wiki_repository return if project.wiki.repository_exists? @@ -125,7 +99,7 @@ module Github label.color = representation.color end - cached[:label_ids][label.title] = label.id + cached[:label_ids][representation.title] = label.id rescue => e error(:label, representation.url, e.message) end @@ -176,7 +150,9 @@ module Github next unless merge_request.new_record? && pull_request.valid? begin - pull_request.restore_branches! + # If the PR has been created/updated after we last fetched the + # remote, we fetch again to get the up-to-date refs. + fetch_remote if pull_request.updated_at > last_fetched_at author_id = user_id(pull_request.author, project.creator_id) description = format_description(pull_request.description, pull_request.author) @@ -185,6 +161,7 @@ module Github iid: pull_request.iid, title: pull_request.title, description: description, + ref_fetched: true, source_project: pull_request.source_project, source_branch: pull_request.source_branch_name, source_branch_sha: pull_request.source_branch_sha, @@ -202,17 +179,10 @@ module Github merge_request.save!(validate: false) merge_request.merge_request_diffs.create - # Fetch review comments review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - - # Fetch comments - comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" - fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) - ensure - pull_request.remove_restored_branches! end end @@ -241,12 +211,17 @@ module Github # for both features, like manipulating assignees, labels # and milestones, are provided within the Issues API. if representation.pull_request? - return unless representation.has_labels? + return unless representation.labels? || representation.comments? merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + + if representation.labels? + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + end + + fetch_comments_conditionally(merge_request, representation) else - return if Issue.where(iid: representation.iid, project_id: project.id).exists? + return if Issue.exists?(iid: representation.iid, project_id: project.id) author_id = user_id(representation.author, project.creator_id) issue = Issue.new @@ -255,25 +230,30 @@ module Github issue.title = representation.title issue.description = format_description(representation.description, representation.author) issue.state = representation.state - issue.label_ids = label_ids(representation.labels) issue.milestone_id = milestone_id(representation.milestone) issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] issue.created_at = representation.created_at issue.updated_at = representation.updated_at issue.save!(validate: false) - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end + issue.update( + label_ids: label_ids(representation.labels), + assignee_ids: assignee_ids(representation.assignees)) + + fetch_comments_conditionally(issue, representation) end rescue => e error(:issue, representation.url, e.message) end end + def fetch_comments_conditionally(issuable, representation) + if representation.comments? + comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" + fetch_comments(issuable, :comment, comments_url) + end + end + def fetch_comments(noteable, type, url, klass = Note) while url comments = Github::Client.new(options).get(url) @@ -332,7 +312,11 @@ module Github end def label_ids(labels) - labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + labels.map { |label| cached[:label_ids][label.title] }.compact + end + + def assignee_ids(assignees) + assignees.map { |assignee| user_id(assignee) }.compact end def milestone_id(milestone) diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb new file mode 100644 index 00000000000..171f0872666 --- /dev/null +++ b/lib/github/import/issue.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb new file mode 100644 index 00000000000..18adff560b6 --- /dev/null +++ b/lib/github/import/legacy_diff_note.rb @@ -0,0 +1,12 @@ +module Github + class Import + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb new file mode 100644 index 00000000000..c258e5d5e0e --- /dev/null +++ b/lib/github/import/merge_request.rb @@ -0,0 +1,13 @@ +module Github + class Import + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb new file mode 100644 index 00000000000..8cf4f30e6b7 --- /dev/null +++ b/lib/github/import/note.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Note < ::Note + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index 823e8e9a9c4..0087a3d3c4f 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -7,10 +7,14 @@ module Github raw.dig('user', 'login') || 'unknown' end + def repo? + raw['repo'].present? + end + def repo - return @repo if defined?(@repo) + return unless repo? - @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + @repo ||= Github::Representation::Repo.new(raw['repo']) end def ref @@ -25,10 +29,6 @@ module Github Commit.truncate_sha(sha) end - def exists? - @exists ||= branch_exists? && commit_exists? - end - def valid? sha.present? && ref.present? end @@ -47,14 +47,6 @@ module Github private - def branch_exists? - repository.branch_exists?(ref) - end - - def commit_exists? - repository.branch_names_contains(sha).include?(ref) - end - def repository @repository ||= options.fetch(:repository) end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 1b5be91461b..83bf0b5310d 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -23,7 +23,7 @@ module Github private def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb index 9713b82615d..768ba3b993c 100644 --- a/lib/github/representation/issuable.rb +++ b/lib/github/representation/issuable.rb @@ -23,14 +23,14 @@ module Github @author ||= Github::Representation::User.new(raw['user'], options) end - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) + def labels? + raw['labels'].any? end - def assigned? - raw['assignee'].present? + def labels + @labels ||= Array(raw['labels']).map do |label| + Github::Representation::Label.new(label, options) + end end end end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index df3540a6e6c..4f1a02cb90f 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -1,25 +1,27 @@ module Github module Representation class Issue < Representation::Issuable - def labels - raw['labels'] - end - def state raw['state'] == 'closed' ? 'closed' : 'opened' end - def has_comments? + def comments? raw['comments'] > 0 end - def has_labels? - labels.count > 0 - end - def pull_request? raw['pull_request'].present? end + + def assigned? + raw['assignees'].present? + end + + def assignees + @assignees ||= Array(raw['assignees']).map do |user| + Github::Representation::User.new(user, options) + end + end end end end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 55461097e8a..0171179bb0f 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -1,26 +1,17 @@ module Github module Representation class PullRequest < Representation::Issuable - delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true - delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + delegate :sha, to: :source_branch, prefix: true + delegate :sha, to: :target_branch, prefix: true def source_project project end def source_branch_name - @source_branch_name ||= - if cross_project? || !source_branch_exists? - source_branch_name_prefixed - else - source_branch_ref - end - end - - def source_branch_exists? - return @source_branch_exists if defined?(@source_branch_exists) - - @source_branch_exists = !cross_project? && source_branch.exists? + # Mimic the "user:branch" displayed in the MR widget, + # i.e. "Request to merge rymai:add-external-mounts into master" + cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref end def target_project @@ -28,11 +19,7 @@ module Github end def target_branch_name - @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed - end - - def target_branch_exists? - @target_branch_exists ||= target_branch.exists? + target_branch.ref end def state @@ -50,16 +37,14 @@ module Github source_branch.valid? && target_branch.valid? end - def restore_branches! - restore_source_branch! - restore_target_branch! + def assigned? + raw['assignee'].present? end - def remove_restored_branches! - return if opened? + def assignee + return unless assigned? - remove_source_branch! - remove_target_branch! + @assignee ||= Github::Representation::User.new(raw['assignee'], options) end private @@ -72,48 +57,14 @@ module Github @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) end - def source_branch_name_prefixed - "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" - end - def target_branch @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) end - def target_branch_name_prefixed - "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" - end - def cross_project? - return true if source_branch_repo.nil? - - source_branch_repo.id != target_branch_repo.id - end - - def restore_source_branch! - return if source_branch_exists? - - source_branch.restore!(source_branch_name) - end - - def restore_target_branch! - return if target_branch_exists? - - target_branch.restore!(target_branch_name) - end - - def remove_source_branch! - # We should remove the source/target branches only if they were - # restored. Otherwise, we'll remove branches like 'master' that - # target_branch_exists? returns true. In other words, we need - # to clean up only the restored branches that (source|target)_branch_exists? - # returns false for the first time it has been called, because of - # this that is important to memoize these values. - source_branch.remove!(source_branch_name) unless source_branch_exists? - end + return true unless source_branch.repo? - def remove_target_branch! - target_branch.remove!(target_branch_name) unless target_branch_exists? + source_branch.repo.id != target_branch.repo.id end end end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb new file mode 100644 index 00000000000..c88eb9783ed --- /dev/null +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -0,0 +1,65 @@ +module Gitlab + module BackgroundMigration + class CreateForkNetworkMembershipsRange + RESCHEDULE_DELAY = 15 + + class ForkedProjectLink < ActiveRecord::Base + self.table_name = 'forked_project_links' + end + + def perform(start_id, end_id) + log("Creating memberships for forks: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS + INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) + + SELECT fork_network_members.fork_network_id, + forked_project_links.forked_to_project_id, + forked_project_links.forked_from_project_id + + FROM forked_project_links + + INNER JOIN fork_network_members + ON forked_project_links.forked_from_project_id = fork_network_members.project_id + + WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT true + FROM fork_network_members existing_members + WHERE existing_members.project_id = forked_project_links.forked_to_project_id + ) + INSERT_MEMBERS + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def missing_members?(start_id, end_id) + count_sql = <<~MISSING_MEMBERS + SELECT COUNT(*) + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE forked_project_links.forked_from_project_id = projects.id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + MISSING_MEMBERS + + ForkNetworkMember.count_by_sql(count_sql) > 0 + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb new file mode 100644 index 00000000000..e94719db72e --- /dev/null +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -0,0 +1,53 @@ +class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys + class GpgKey < ActiveRecord::Base + self.table_name = 'gpg_keys' + + include EachBatch + include ShaAttribute + + sha_attribute :primary_keyid + sha_attribute :fingerprint + + has_many :subkeys, class_name: 'GpgKeySubkey' + end + + class GpgKeySubkey < ActiveRecord::Base + self.table_name = 'gpg_key_subkeys' + + include ShaAttribute + + sha_attribute :keyid + sha_attribute :fingerprint + end + + def perform(gpg_key_id) + gpg_key = GpgKey.find_by(id: gpg_key_id) + + return if gpg_key.nil? + return if gpg_key.subkeys.any? + + create_subkeys(gpg_key) + update_signatures(gpg_key) + end + + private + + def create_subkeys(gpg_key) + gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) + + gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| + gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) + end + + # Improve latency by doing all INSERTs in a single call + GpgKey.transaction do + gpg_key.save! + end + end + + def update_signatures(gpg_key) + return unless gpg_key.subkeys.exists? + + InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) + end +end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 3fde1b09efb..8e5c95f2287 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -3,11 +3,18 @@ module Gitlab class DeserializeMergeRequestDiffsAndCommits attr_reader :diff_ids, :commit_rows, :file_rows + class Error < StandardError + def backtrace + cause.backtrace + end + end + class MergeRequestDiff < ActiveRecord::Base self.table_name = 'merge_request_diffs' end BUFFER_ROWS = 1000 + DIFF_FILE_BUFFER_ROWS = 100 def perform(start_id, stop_id) merge_request_diffs = MergeRequestDiff @@ -26,13 +33,17 @@ module Gitlab if diff_ids.length > BUFFER_ROWS || commit_rows.length > BUFFER_ROWS || - file_rows.length > BUFFER_ROWS + file_rows.length > DIFF_FILE_BUFFER_ROWS flush_buffers! end end flush_buffers! + rescue => e + Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") + + raise Error.new(e.inspect) end private @@ -45,17 +56,28 @@ module Gitlab def flush_buffers! if diff_ids.any? - MergeRequestDiff.transaction do - Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) - Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| + bulk_insert('merge_request_diff_commits', commit_rows_slice) + end - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| + bulk_insert('merge_request_diff_files', file_rows_slice) end + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) end reset_buffers! end + def bulk_insert(table, rows) + Gitlab::Database.bulk_insert(table, rows) + rescue ActiveRecord::RecordNotUnique + ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort + + Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") + end + def single_diff_rows(merge_request_diff) sha_attribute = Gitlab::Database::ShaAttribute.new commits = YAML.load(merge_request_diff.st_commits) rescue [] diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb new file mode 100644 index 00000000000..bc53e6d7f94 --- /dev/null +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -0,0 +1,313 @@ +module Gitlab + module BackgroundMigration + class NormalizeLdapExternUidsRange + class Identity < ActiveRecord::Base + self.table_name = 'identities' + end + + # Copied this class to make this migration resilient to future code changes. + # And if the normalize behavior is changed in the future, it must be + # accompanied by another migration. + module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end + + def perform(start_id, end_id) + return unless migrate? + + ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) + ldap_identities.each do |identity| + begin + identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + unless identity.save + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." + end + rescue Gitlab::LDAP::DN::FormatError => e + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." + end + end + end + + def migrate? + Identity.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb new file mode 100644 index 00000000000..2ef3a207dd3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -0,0 +1,59 @@ +module Gitlab + module BackgroundMigration + class PopulateForkNetworksRange + def perform(start_id, end_id) + log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_from_project_id + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + + log("Creating memberships for root projects: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_ROOT + INSERT INTO fork_network_members (fork_network_id, project_id) + SELECT DISTINCT fork_networks.id, fork_networks.root_project_id + + FROM fork_networks + + INNER JOIN forked_project_links + ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = fork_networks.root_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_ROOT + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY + BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 9323bfc7fb2..1d98d187805 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -56,7 +56,8 @@ module Gitlab name: project_path, path: project_path, repository_storage: storage_name, - namespace_id: group&.id + namespace_id: group&.id, + skip_disk_validation: true } project = Projects::CreateService.new(user, project_params).execute diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 28bbf3b384e..033ecd15749 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,16 +149,21 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, + source_branch_sha: source_branch_sha, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, + target_branch_sha: target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, @@ -236,7 +241,7 @@ module Gitlab end def generate_line_code(pr_comment) - Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + Gitlab::Git.diff_line_code(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) end def pull_request_comment_attributes(comment) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index ad78ae244b2..72b75791bbb 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -155,7 +155,9 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) + if s.scan(Gitlab::Regex.build_trace_section_regex) + handle_section(s) + elsif s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) elsif s.scan(/\e(([@-_])(.*?)?)?$/) break @@ -183,6 +185,15 @@ module Gitlab ) end + def handle_section(s) + action = s[1] + timestamp = s[2] + section = s[3] + line = s.matched()[0...-5] # strips \r\033[0K + + @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + end + def handle_sequence(s) indicator = s[1] commands = s[2].split ';' diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb new file mode 100644 index 00000000000..8d82e1b288d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Base + attr_reader :pipeline, :project, :current_user + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + + @project = command.project + @current_user = command.current_user + end + + def perform! + raise NotImplementedError + end + + def break? + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb new file mode 100644 index 00000000000..d5e17a123df --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Create < Chain::Base + include Chain::Helpers + + def perform! + ::Ci::Pipeline.transaction do + pipeline.save! + + @command.seeds_block&.call(pipeline) + + ::Ci::CreatePipelineStagesService + .new(project, current_user) + .execute(pipeline) + end + rescue ActiveRecord::RecordInvalid => e + error("Failed to persist the pipeline: #{e}") + end + + def break? + !pipeline.persisted? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb new file mode 100644 index 00000000000..02d81286f21 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Helpers + def branch_exists? + return @is_branch if defined?(@is_branch) + + @is_branch = project.repository.branch_exists?(pipeline.ref) + end + + def tag_exists? + return @is_tag if defined?(@is_tag) + + @is_tag = project.repository.tag_exists?(pipeline.ref) + end + + def error(message) + pipeline.errors.add(:base, message) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb new file mode 100644 index 00000000000..015f2988327 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -0,0 +1,36 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Sequence + def initialize(pipeline, command, sequence) + @pipeline = pipeline + @completed = [] + + @sequence = sequence.map do |chain| + chain.new(pipeline, command) + end + end + + def build! + @sequence.each do |step| + step.perform! + + break if step.break? + + @completed << step + end + + @pipeline.tap do + yield @pipeline, self if block_given? + end + end + + def complete? + @completed.size == @sequence.size + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb new file mode 100644 index 00000000000..9a72de87bab --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Skip < Chain::Base + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + + def perform! + if skipped? + @pipeline.skip if @command.save_incompleted + end + end + + def skipped? + !@command.ignore_skip_ci && commit_message_skips_ci? + end + + def break? + skipped? + end + + private + + def commit_message_skips_ci? + return false unless @pipeline.git_commit_message + + @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb new file mode 100644 index 00000000000..4913a604079 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Abilities < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + + def perform! + unless project.builds_enabled? + return error('Pipelines are disabled!') + end + + unless allowed_to_trigger_pipeline? + if can?(current_user, :create_pipeline, project) + return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + else + return error('Insufficient permissions to create a new pipeline') + end + end + end + + def break? + @pipeline.errors.any? + end + + def allowed_to_trigger_pipeline? + if current_user + allowed_to_create? + else # legacy triggers don't have a corresponding user + !project.protected_for?(@pipeline.ref) + end + end + + def allowed_to_create? + return unless can?(current_user, :create_pipeline, project) + + access = Gitlab::UserAccess.new(current_user, project: project) + + if branch_exists? + access.can_update_branch?(@pipeline.ref) + elsif tag_exists? + access.can_create_tag?(@pipeline.ref) + else + true # Allow it for now and we'll reject when we check ref existence + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb new file mode 100644 index 00000000000..075504bcce5 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Config < Chain::Base + include Chain::Helpers + + def perform! + unless @pipeline.config_processor + unless @pipeline.ci_yaml_file + return error("Missing #{@pipeline.ci_yaml_file_path} file") + end + + if @command.save_incompleted && @pipeline.has_yaml_errors? + @pipeline.drop!(:config_error) + end + + return error(@pipeline.yaml_errors) + end + + unless @pipeline.has_stage_seeds? + return error('No stages / jobs for this pipeline.') + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb new file mode 100644 index 00000000000..70a4cfdbdea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Repository < Chain::Base + include Chain::Helpers + + def perform! + unless branch_exists? || tag_exists? + return error('Reference not found') + end + + ## TODO, we check commit in the service, that is why + # there is no repository access here. + # + unless pipeline.sha + return error('Commit not found') + end + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb new file mode 100644 index 00000000000..469fc094cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -0,0 +1,143 @@ +module Gitlab + module Ci + module Pipeline + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module Duration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb deleted file mode 100644 index 3208cc2bef6..00000000000 --- a/lib/gitlab/ci/pipeline_duration.rb +++ /dev/null @@ -1,141 +0,0 @@ -module Gitlab - module Ci - # # Introduction - total running time - # - # The problem this module is trying to solve is finding the total running - # time amongst all the jobs, excluding retries and pending (queue) time. - # We could reduce this problem down to finding the union of periods. - # - # So each job would be represented as a `Period`, which consists of - # `Period#first` as when the job started and `Period#last` as when the - # job was finished. A simple example here would be: - # - # * A (1, 3) - # * B (2, 4) - # * C (6, 7) - # - # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. - # C begins from 6, and ends to 7. Visually it could be viewed as: - # - # 0 1 2 3 4 5 6 7 - # AAAAAAA - # BBBBBBB - # CCCC - # - # The union of A, B, and C would be (1, 4) and (6, 7), therefore the - # total running time should be: - # - # (4 - 1) + (7 - 6) => 4 - # - # # The Algorithm - # - # The algorithm used here for union would be described as follow. - # First we make sure that all periods are sorted by `Period#first`. - # Then we try to merge periods by iterating through the first period - # to the last period. The goal would be merging all overlapped periods - # so that in the end all the periods are discrete. When all periods - # are discrete, we're free to just sum all the periods to get real - # running time. - # - # Here we begin from A, and compare it to B. We could find that - # before A ends, B already started. That is `B.first <= A.last` - # that is `2 <= 3` which means A and B are overlapping! - # - # When we found that two periods are overlapping, we would need to merge - # them into a new period and disregard the old periods. To make a new - # period, we take `A.first` as the new first because remember? we sorted - # them, so `A.first` must be smaller or equal to `B.first`. And we take - # `[A.last, B.last].max` as the new last because we want whoever ended - # later. This could be broken into two cases: - # - # 0 1 2 3 4 - # AAAAAAA - # BBBBBBB - # - # Or: - # - # 0 1 2 3 4 - # AAAAAAAAAA - # BBBB - # - # So that we need to take whoever ends later. Back to our example, - # after merging and discard A and B it could be visually viewed as: - # - # 0 1 2 3 4 5 6 7 - # DDDDDDDDDD - # CCCC - # - # Now we could go on and compare the newly created D and the old C. - # We could figure out that D and C are not overlapping by checking - # `C.first <= D.last` is `false`. Therefore we need to keep both C - # and D. The example would end here because there are no more jobs. - # - # After having the union of all periods, we just need to sum the length - # of all periods to get total time. - # - # (4 - 1) + (7 - 6) => 4 - # - # That is 4 is the answer in the example. - module PipelineDuration - extend self - - Period = Struct.new(:first, :last) do - def duration - last - first - end - end - - def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.builds.latest - .where(status: status).where.not(started_at: nil).order(:started_at) - - from_builds(builds) - end - - def from_builds(builds) - now = Time.now - - periods = builds.map do |b| - Period.new(b.started_at, b.finished_at || now) - end - - from_periods(periods) - end - - # periods should be sorted by `first` - def from_periods(periods) - process_duration(process_periods(periods)) - end - - private - - def process_periods(periods) - return periods if periods.empty? - - periods.drop(1).inject([periods.first]) do |result, current| - previous = result.last - - if overlap?(previous, current) - result[-1] = merge(previous, current) - result - else - result << current - end - end - end - - def overlap?(previous, current) - current.first <= previous.last - end - - def merge(previous, current) - Period.new(previous.first, [previous.last, current.last].max) - end - - def process_duration(periods) - periods.sum(&:duration) - end - end - end -end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index e19aae35a81..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -3,7 +3,9 @@ module Gitlab module Stage class Seed attr_reader :pipeline + delegate :project, to: :pipeline + delegate :size, to: :@jobs def initialize(pipeline, stage, jobs) @pipeline = pipeline diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b835bb669a..baf55b1fa07 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -27,6 +27,12 @@ module Gitlab end end + def extract_sections + read do |stream| + stream.extract_sections + end + end + def set(data) write do |stream| data = job.hide_secrets(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb new file mode 100644 index 00000000000..9bb0166c9e3 --- /dev/null +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -0,0 +1,97 @@ +module Gitlab + module Ci + class Trace + class SectionParser + def initialize(lines) + @lines = lines + end + + def parse! + @markers = {} + + @lines.each do |line, pos| + parse_line(line, pos) + end + end + + def sections + sanitize_markers.map do |name, markers| + start_, end_ = markers + + { + name: name, + byte_start: start_[:marker], + byte_end: end_[:marker], + date_start: start_[:timestamp], + date_end: end_[:timestamp] + } + end + end + + private + + def parse_line(line, line_start_position) + s = StringScanner.new(line) + until s.eos? + find_next_marker(s) do |scanner| + marker_begins_at = line_start_position + scanner.pointer + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + marker_ends_at = line_start_position + scanner.pointer + handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at) + true + else + false + end + end + end + end + + def sanitize_markers + @markers.select do |_, markers| + markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end + end + end + + def handle_line(action, time, name, marker_start, marker_end) + action = action.to_sym + timestamp = Time.at(time).utc + marker = if action == :start + marker_end + else + marker_start + end + + @markers[name] ||= [] + @markers[name] << { + name: name, + action: action, + timestamp: timestamp, + marker: marker + } + end + + def beginning_of_section_regex + @beginning_of_section_regex ||= /section_/.freeze + end + + def find_next_marker(s) + beginning_of_section_len = 8 + maybe_marker = s.exist?(beginning_of_section_regex) + + if maybe_marker.nil? + s.terminate + else + # repositioning at the beginning of the match + s.pos += maybe_marker - beginning_of_section_len + if block_given? + good_marker = yield(s) + # if not a good marker: Consuming the matched beginning_of_section_regex + s.pos += beginning_of_section_len unless good_marker + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index ab3408f48d6..d52194f688b 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -90,8 +90,25 @@ module Gitlab # so we just silently ignore error for now end + def extract_sections + return [] unless valid? + + lines = to_enum(:each_line_with_pos) + parser = SectionParser.new(lines) + + parser.parse! + parser.sections + end + private + def each_line_with_pos + stream.seek(0, IO::SEEK_SET) + stream.each_line do |line| + yield [line, stream.pos - line.bytesize] + end + end + def read_last_lines(limit) to_enum(:reverse_line).first(limit).reverse.join end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 243c1f1394d..7e7aaeeaa17 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -23,7 +23,8 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - @extractor.project.forked_from?(issue.project) # Don't extract issues on original project + # Don't extract issues from the project this project was forked from + @extractor.project.forked_from?(issue.project) end end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 98dfe900044..2a0cb640a14 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,82 +4,29 @@ module Gitlab include Gitlab::Routing include IconsHelper - MissingResolution = Class.new(ResolutionError) - CONTEXT_LINES = 3 - attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository - - def initialize(merge_file_result, conflict, merge_request:) - @merge_file_result = merge_file_result - @their_path = conflict[:theirs][:path] - @our_path = conflict[:ours][:path] - @our_mode = conflict[:ours][:mode] - @merge_request = merge_request - @repository = merge_request.project.repository - @match_line_headers = {} - end - - def content - merge_file_result[:data] - end + attr_reader :merge_request - def our_blob - @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) - end + # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps + attr_reader :raw - def type - lines unless @type + delegate :type, :content, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw - @type.inquiry + def initialize(raw, merge_request:) + @raw = raw + @merge_request = merge_request + @match_line_headers = {} end - # Array of Gitlab::Diff::Line objects def lines return @lines if defined?(@lines) - begin - @type = 'text' - @lines = Gitlab::Conflict::Parser.new.parse(content, - our_path: our_path, - their_path: their_path, - parent_file: self) - rescue Gitlab::Conflict::Parser::ParserError - @type = 'text-editor' - @lines = nil - end + @lines = raw.lines.nil? ? nil : map_raw_lines(raw.lines) end def resolve_lines(resolution) - section_id = nil - - lines.map do |line| - unless line.type - section_id = nil - next line - end - - section_id ||= line_code(line) - - case resolution[section_id] - when 'head' - next unless line.type == 'new' - when 'origin' - next unless line.type == 'old' - else - raise MissingResolution, "Missing resolution for section ID: #{section_id}" - end - - line - end.compact - end - - def resolve_content(resolution) - if resolution == content - raise MissingResolution, "Resolved content has no changes for file #{our_path}" - end - - resolution + map_raw_lines(raw.resolve_lines(resolution)) end def highlight_lines! @@ -163,7 +110,7 @@ module Gitlab end def line_code(line) - Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(our_path, line.new_pos, line.old_pos) end def create_match_line(line) @@ -227,15 +174,14 @@ module Gitlab new_path: our_path) end - # Don't try to print merge_request or repository. - def inspect - instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| - value = instance_variable_get("@#{instance_variable}") + private - "#{instance_variable}=\"#{value}\"" + def map_raw_lines(raw_lines) + raw_lines.map do |raw_line| + Gitlab::Diff::Line.new(raw_line[:full_line], raw_line[:type], + raw_line[:line_obj_index], raw_line[:line_old], + raw_line[:line_new], parent_file: self) end - - "#<#{self.class} #{instance_variables.join(' ')}>" end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 90f83e0f810..fb28e80ff73 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,48 +1,29 @@ module Gitlab module Conflict class FileCollection - ConflictSideMissing = Class.new(StandardError) - - attr_reader :merge_request, :our_commit, :their_commit, :project - - delegate :repository, to: :project - - class << self - # We can only write when getting the merge index from the source - # project, because we will write to that project. We don't use this all - # the time because this fetches a ref into the source project, which - # isn't needed for reading. - def for_resolution(merge_request) - project = merge_request.source_project - - new(merge_request, project).tap do |file_collection| - project - .repository - .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do - - yield file_collection - end - end - end - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - def read_only(merge_request) - new(merge_request, merge_request.target_project) - end + attr_reader :merge_request, :resolver + + def initialize(merge_request) + source_repo = merge_request.source_project.repository.raw + our_commit = merge_request.source_branch_head.raw + their_commit = merge_request.target_branch_head.raw + target_repo = merge_request.target_project.repository.raw + @resolver = Gitlab::Git::Conflict::Resolver.new(source_repo, our_commit, target_repo, their_commit) + @merge_request = merge_request end - def merge_index - @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + def resolve(user, commit_message, files) + args = { + source_branch: merge_request.source_branch, + target_branch: merge_request.target_branch, + commit_message: commit_message || default_commit_message + } + resolver.resolve_conflicts(user, files, args) end def files - @files ||= merge_index.conflicts.map do |conflict| - raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] - - Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), - conflict, - merge_request: merge_request) + @files ||= resolver.conflicts.map do |conflict_file| + Gitlab::Conflict::File.new(conflict_file, merge_request: merge_request) end end @@ -61,8 +42,8 @@ module Gitlab end def default_commit_message - conflict_filenames = merge_index.conflicts.map do |conflict| - "# #{conflict[:ours][:path]}" + conflict_filenames = files.map do |conflict| + "# #{conflict.our_path}" end <<EOM.chomp @@ -72,15 +53,6 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end - - private - - def initialize(merge_request, project) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.rugged_commit - @their_commit = merge_request.target_branch_head.raw.rugged_commit - @project = project - end end end end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb deleted file mode 100644 index e3678c914db..00000000000 --- a/lib/gitlab/conflict/parser.rb +++ /dev/null @@ -1,74 +0,0 @@ -module Gitlab - module Conflict - class Parser - UnresolvableError = Class.new(StandardError) - UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) - - # Recoverable errors - the conflict can be resolved in an editor, but not with - # sections. - ParserError = Class.new(StandardError) - UnexpectedDelimiter = Class.new(ParserError) - MissingEndDelimiter = Class.new(ParserError) - - def parse(text, our_path:, their_path:, parent_file: nil) - validate_text!(text) - - line_obj_index = 0 - line_old = 1 - line_new = 1 - type = nil - lines = [] - conflict_start = "<<<<<<< #{our_path}" - conflict_middle = '=======' - conflict_end = ">>>>>>> #{their_path}" - - text.each_line.map do |line| - full_line = line.delete("\n") - - if full_line == conflict_start - validate_delimiter!(type.nil?) - - type = 'new' - elsif full_line == conflict_middle - validate_delimiter!(type == 'new') - - type = 'old' - elsif full_line == conflict_end - validate_delimiter!(type == 'old') - - type = nil - elsif line[0] == '\\' - type = 'nonewline' - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - else - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - line_old += 1 if type != 'new' - line_new += 1 if type != 'old' - - line_obj_index += 1 - end - end - - raise MissingEndDelimiter unless type.nil? - - lines - end - - private - - def validate_text!(text) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? - end - - def validate_delimiter!(condition) - raise UnexpectedDelimiter unless condition - end - end - end -end diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb deleted file mode 100644 index 0b61256b35a..00000000000 --- a/lib/gitlab/conflict/resolution_error.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module Conflict - ResolutionError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 31a46a738c3..c169c8fe135 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -86,7 +86,7 @@ module Gitlab user_name: user.name, user_username: user.username, user_email: user.email, - user_avatar: user.avatar_url, + user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index a6ec75da385..357f16936c6 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -29,6 +29,15 @@ module Gitlab adapter_name.casecmp('postgresql').zero? end + # Overridden in EE + def self.read_only? + false + end + + def self.read_write? + !self.read_only? + end + def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 371cbe04b9b..c98eefbce25 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - base_sha == other.base_sha && - start_sha == other.start_sha && - head_sha == other.head_sha + shas_equal?(base_sha, other.base_sha) && + shas_equal?(start_sha, other.start_sha) && + shas_equal?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,6 +47,22 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end + + private + + def shas_equal?(sha1, sha2) + return true if sha1 == sha2 + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + length = [sha1.length, sha2.length].min + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + return false if length < Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index fcac85ff892..ea5891a028a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -27,22 +27,29 @@ module Gitlab @fallback_diff_refs = fallback_diff_refs end - def position(line) + def position(position_marker, position_type: :text) return unless diff_refs - Position.new( + data = { + diff_refs: diff_refs, + position_type: position_type.to_s, old_path: old_path, - new_path: new_path, - old_line: line.old_line, - new_line: line.new_line, - diff_refs: diff_refs - ) + new_path: new_path + } + + if position_type == :text + data.merge!(text_position_properties(position_marker)) + else + data.merge!(image_position_properties(position_marker)) + end + + Position.new(data) end def line_code(line) return if line.meta? - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def line_for_line_code(code) @@ -228,6 +235,14 @@ module Gitlab private + def text_position_properties(line) + { old_line: line.old_line, new_line: line.new_line } + end + + def image_position_properties(image_point) + image_point.to_h + end + def blobs_changed? old_blob && new_blob && old_blob.id != new_blob.id end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb new file mode 100644 index 00000000000..5e923b9e602 --- /dev/null +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -0,0 +1,61 @@ +module Gitlab + module Diff + module Formatters + class BaseFormatter + attr_reader :old_path + attr_reader :new_path + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + attr_reader :position_type + + def initialize(attrs) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + + def key + [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || "")] + end + + def to_h + { + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha, + old_path: old_path, + new_path: new_path, + position_type: position_type + } + end + + def position_type + raise NotImplementedError + end + + def ==(other) + raise NotImplementedError + end + + def complete? + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb new file mode 100644 index 00000000000..ccd0d309972 --- /dev/null +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Diff + module Formatters + class ImageFormatter < BaseFormatter + attr_reader :width + attr_reader :height + attr_reader :x + attr_reader :y + + def initialize(attrs) + @x = attrs[:x] + @y = attrs[:y] + @width = attrs[:width] + @height = attrs[:height] + + super(attrs) + end + + def key + @key ||= super.push(x, y) + end + + def complete? + x && y && width && height + end + + def to_h + super.merge(width: width, height: height, x: x, y: y) + end + + def position_type + "image" + end + + def ==(other) + other.is_a?(self.class) && + x == other.x && + y == other.y + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb new file mode 100644 index 00000000000..01c7e9f51ab --- /dev/null +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -0,0 +1,49 @@ +module Gitlab + module Diff + module Formatters + class TextFormatter < BaseFormatter + attr_reader :old_line + attr_reader :new_line + + def initialize(attrs) + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + super(attrs) + end + + def key + @key ||= super.push(old_line, new_line) + end + + def complete? + old_line || new_line + end + + def to_h + super.merge(old_line: old_line, new_line: new_line) + end + + def line_age + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def position_type + "text" + end + + def ==(other) + other.is_a?(self.class) && + new_line == other.new_line && + old_line == other.old_line + end + end + end + end +end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb new file mode 100644 index 00000000000..65332dfd239 --- /dev/null +++ b/lib/gitlab/diff/image_point.rb @@ -0,0 +1,23 @@ +module Gitlab + module Diff + class ImagePoint + attr_reader :width, :height, :x, :y + + def initialize(width, height, x, y) + @width = width + @height = height + @x = x + @y = y + end + + def to_h + { + width: width, + height: height, + x: x, + y: y + } + end + end + end +end diff --git a/lib/gitlab/diff/line_code.rb b/lib/gitlab/diff/line_code.rb deleted file mode 100644 index f3578ab3d35..00000000000 --- a/lib/gitlab/diff/line_code.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Diff - class LineCode - def self.generate(file_path, new_line_position, old_line_position) - "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" - end - end - end -end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 742f989c50b..7dc9cc7c281 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,7 +17,9 @@ module Gitlab # without having to instantiate all the others that come after it. Enumerator.new do |yielder| @lines.each do |line| - next if filename?(line) + # We're expecting a filename parameter only in a meta-part of the diff content + # when type is defined then we're already in a content-part + next if filename?(line) && type.nil? full_line = line.delete("\n") diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f80afb20f0c..bd0a9502a5e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,37 +1,25 @@ -# Defines a specific location, identified by paths and line numbers, +# Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab module Diff class Position - attr_reader :old_path - attr_reader :new_path - attr_reader :old_line - attr_reader :new_line - attr_reader :base_sha - attr_reader :start_sha - attr_reader :head_sha - + attr_accessor :formatter + + delegate :old_path, + :new_path, + :base_sha, + :start_sha, + :head_sha, + :old_line, + :new_line, + :position_type, to: :formatter + + # A position can belong to a text line or to an image coordinate + # it depends of the position_type argument. + # Text position will have: new_line and old_line + # Image position will have: width, height, x, y def initialize(attrs = {}) - if diff_file = attrs[:diff_file] - attrs[:diff_refs] = diff_file.diff_refs - attrs[:old_path] = diff_file.old_path - attrs[:new_path] = diff_file.new_path - end - - if diff_refs = attrs[:diff_refs] - attrs[:base_sha] = diff_refs.base_sha - attrs[:start_sha] = diff_refs.start_sha - attrs[:head_sha] = diff_refs.head_sha - end - - @old_path = attrs[:old_path] - @new_path = attrs[:new_path] - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - - @old_line = attrs[:old_line] - @new_line = attrs[:new_line] + @formatter = get_formatter_class(attrs[:position_type]).new(attrs) end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -46,27 +34,23 @@ module Gitlab end def encode_with(coder) - coder['attributes'] = self.to_h + coder['attributes'] = formatter.to_h end def key - @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + formatter.key end def ==(other) - other.is_a?(self.class) && key == other.key + other.is_a?(self.class) && + other.diff_refs == diff_refs && + other.old_path == old_path && + other.new_path == new_path && + other.formatter == formatter end def to_h - { - old_path: old_path, - new_path: new_path, - old_line: old_line, - new_line: new_line, - base_sha: base_sha, - start_sha: start_sha, - head_sha: head_sha - } + formatter.to_h end def inspect @@ -74,23 +58,15 @@ module Gitlab end def complete? - file_path.present? && - (old_line || new_line) && - diff_refs.complete? + file_path.present? && formatter.complete? && diff_refs.complete? end def to_json(opts = nil) - JSON.generate(self.to_h, opts) + JSON.generate(formatter.to_h, opts) end def type - if old_line && new_line - nil - elsif new_line - 'new' - else - 'old' - end + formatter.line_age end def unchanged? @@ -149,6 +125,17 @@ module Gitlab diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end + + def get_formatter_class(type) + type ||= "text" + + case type + when 'image' + Gitlab::Diff::Formatters::ImageFormatter + else + Gitlab::Diff::Formatters::TextFormatter + end + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index c5a8ea12245..c4c60d1dfee 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,7 +2,7 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze + DEFAULT_CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze @@ -20,7 +20,7 @@ module Gitlab attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found attr_reader :failed_files - def initialize(branch:, ce_repo: CE_REPO) + def initialize(branch:, ce_repo: DEFAULT_CE_REPO) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @@ -132,7 +132,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index a8cb7fc3fe7..0e9ef4f897c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,31 +6,33 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme/i, - changelog: /\A(changelog|history|changes|news)/i, - license: /\A(licen[sc]e|copying)(\..+|\z)/i, - contributing: /\Acontributing/i, + readme: /\Areadme[^\/]*\z/i, + changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, + license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, + contributing: /\Acontributing[^\/]*\z/i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, + issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, + merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - route_map: 'route-map.yml', + route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile/, + cartfile: /\ACartfile[^\/]*\z/, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\.gemspec\z/, + gemspec: /\A[^\/]*\.gemspec\z/, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\.podspec\.json\z/, - podspec: /\.podspec\z/, - requirements_txt: /requirements\.txt\z/, + podspec_json: /\A[^\/]*\.podspec\.json\z/, + podspec: /\A[^\/]*\.podspec\z/, + requirements_txt: /\A[^\/]*requirements\.txt\z/, yarn_lock: 'yarn.lock' }.freeze @@ -63,13 +65,11 @@ module Gitlab # type_of('README.md') # => :readme # type_of('VERSION') # => :version def self.type_of(path) - name = File.basename(path) - PATTERNS.each do |type, search| did_match = if search.is_a?(Regexp) - name =~ search + path =~ search else - name.casecmp(search) == 0 + path.casecmp(search) == 0 end return type if did_match diff --git a/lib/gitlab/gcp/model.rb b/lib/gitlab/gcp/model.rb new file mode 100644 index 00000000000..195391f0e3c --- /dev/null +++ b/lib/gitlab/gcp/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Gcp + module Model + def table_name_prefix + "gcp_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c78fe63f9b5..1f31cdbc96d 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -66,6 +66,10 @@ module Gitlab end end end + + def diff_line_code(file_path, new_line_position, old_line_position) + "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" + end end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb new file mode 100644 index 00000000000..fc1595f1faf --- /dev/null +++ b/lib/gitlab/git/conflict/file.rb @@ -0,0 +1,86 @@ +module Gitlab + module Git + module Conflict + class File + attr_reader :content, :their_path, :our_path, :our_mode, :repository + + def initialize(repository, commit_oid, conflict, content) + @repository = repository + @commit_oid = commit_oid + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @content = content + end + + def lines + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Git::Conflict::Parser.parse(content, + our_path: our_path, + their_path: their_path) + rescue Gitlab::Git::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end + end + + def type + lines unless @type + + @type.inquiry + end + + def our_blob + # REFACTOR NOTE: the source of `commit_oid` used to be + # `merge_request.diff_refs.head_sha`. Instead of passing this value + # around the new lib structure, I decided to use `@commit_oid` which is + # equivalent to `merge_request.source_branch_head.raw.rugged_commit.oid`. + # That is what `merge_request.diff_refs.head_sha` is equivalent to when + # `merge_request` is not persisted (see `MergeRequest#diff_head_commit`). + # I think using the same oid is more consistent anyways, but if Conflicts + # start breaking, the change described above is a good place to look at. + @our_blob ||= repository.blob_at(@commit_oid, our_path) + end + + def line_code(line) + Gitlab::Git.diff_line_code(our_path, line[:line_new], line[:line_old]) + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line[:type] + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line[:type] == 'new' + when 'origin' + next unless line[:type] == 'old' + else + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def resolve_content(resolution) + if resolution == content + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb new file mode 100644 index 00000000000..3effa9d2d31 --- /dev/null +++ b/lib/gitlab/git/conflict/parser.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Parser + UnresolvableError = Class.new(StandardError) + UnmergeableFile = Class.new(UnresolvableError) + UnsupportedEncoding = Class.new(UnresolvableError) + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + ParserError = Class.new(StandardError) + UnexpectedDelimiter = Class.new(ParserError) + MissingEndDelimiter = Class.new(ParserError) + + class << self + def parse(text, our_path:, their_path:, parent_file: nil) + validate_text!(text) + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + validate_delimiter!(type.nil?) + + type = 'new' + elsif full_line == conflict_middle + validate_delimiter!(type == 'new') + + type = 'old' + elsif full_line == conflict_end + validate_delimiter!(type == 'old') + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + else + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb new file mode 100644 index 00000000000..df509c5f4ce --- /dev/null +++ b/lib/gitlab/git/conflict/resolver.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Resolver + ConflictSideMissing = Class.new(StandardError) + ResolutionError = Class.new(StandardError) + + def initialize(repository, our_commit, target_repository, their_commit) + @repository = repository + @our_commit = our_commit.rugged_commit + @target_repository = target_repository + @their_commit = their_commit.rugged_commit + end + + def conflicts + @conflicts ||= begin + target_index = @target_repository.rugged.merge_commits(@our_commit, @their_commit) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + target_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Git::Conflict::File.new( + @target_repository, + @our_commit.oid, + conflict, + target_index.merge_file(conflict[:ours][:path])[:data] + ) + end + end + end + + def resolve_conflicts(user, files, source_branch:, target_branch:, commit_message:) + @repository.with_repo_branch_commit(@target_repository, target_branch) do + files.each do |file_params| + conflict_file = conflict_for_path(file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: commit_message, + parents: [@our_commit, @their_commit].map(&:oid) + } + + @repository.commit_index(user, source_branch, index, commit_params) + end + end + + def conflict_for_path(old_path, new_path) + conflicts.find do |conflict| + conflict.their_path == old_path && conflict.our_path == new_path + end + end + + private + + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def index + @index ||= @repository.rugged.merge_commits(@our_commit, @their_commit) + end + + def write_resolved_file_to_index(file, params) + if params[:sections] + resolved_lines = file.resolve_lines(params[:sections]) + new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") + + new_file << "\n" if file.our_blob.data.ends_with?("\n") + elsif params[:content] + new_file = file.resolve_content(params[:content]) + end + + our_path = file.our_path + + index.add(path: our_path, oid: @repository.rugged.write(new_file, :blob), mode: file.our_mode) + index.conflict_remove(our_path) + end + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 096301d300f..ca94b4baa59 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -24,41 +24,13 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze - class << self - # The maximum size of a diff to display. - def size_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit - else - find_size_limit - end - end - - # The maximum size before a diff is collapsed. - def collapse_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit - else - find_collapse_limit - end - end + # The maximum size of a diff to display. + SIZE_LIMIT = 100.kilobytes - def find_size_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 200.kilobytes - else - 100.kilobytes - end - end - - def find_collapse_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 100.kilobytes - else - 10.kilobytes - end - end + # The maximum size before a diff is collapsed. + COLLAPSE_LIMIT = 10.kilobytes + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -172,7 +144,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.size_limit + @too_large = @diff.bytesize >= SIZE_LIMIT else @too_large end @@ -190,7 +162,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end def collapse! @@ -275,14 +247,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= self.class.size_limit + if size >= SIZE_LIMIT too_large! return true end end end - if !expanded && size >= self.class.collapse_limit + if !expanded && size >= COLLAPSE_LIMIT collapse! return true end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index f80193ac553..9d0b47a1a6d 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -11,9 +11,11 @@ module Gitlab # # This class is thread-safe via RequestStore. class Env - WHITELISTED_GIT_VARIABLES = %w[ + WHITELISTED_VARIABLES = %w[ GIT_OBJECT_DIRECTORY + GIT_OBJECT_DIRECTORY_RELATIVE GIT_ALTERNATE_OBJECT_DIRECTORIES + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze def self.set(env) @@ -28,12 +30,23 @@ module Gitlab RequestStore.fetch(:gitlab_git_env) { {} } end + def self.to_env_hash + env = {} + + all.compact.each do |key, value| + value = value.join(File::PATH_SEPARATOR) if value.is_a?(Array) + env[key.to_s] = value + end + + env + end + def self.[](key) all[key] end def self.whitelist_git_env(env) - env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access + env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 208e4bbaf60..e29a1f7afa1 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -22,22 +22,22 @@ module Gitlab File.exist?(path) end - def trigger(gl_id, oldrev, newrev, ref) + def trigger(gl_id, gl_username, oldrev, newrev, ref) return [true, nil] unless exists? Bundler.with_clean_env do case name when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) + call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + call_update_hook(gl_id, gl_username, oldrev, newrev, ref) end end end private - def call_receive_hook(gl_id, oldrev, newrev, ref) + def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") exit_status = false @@ -45,6 +45,7 @@ module Gitlab vars = { 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, 'GL_REPOSITORY' => repository.gl_repository @@ -80,9 +81,13 @@ module Gitlab [exit_status, exit_message] end - def call_update_hook(gl_id, oldrev, newrev, ref) + def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) Dir.chdir(repo_path) do - stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username + } + stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index ea8a87a1290..c327e9b1616 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -5,12 +5,13 @@ module Gitlab attr_accessor :oldrev, :newrev, :ref - def execute(committer, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = committer.gl_id - @oldrev = oldrev - @newrev = newrev - @ref = ref + def execute(pusher, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = pusher.gl_id + @gl_username = pusher.name + @oldrev = oldrev + @newrev = newrev + @ref = ref %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) @@ -29,7 +30,7 @@ module Gitlab def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, oldrev, newrev, ref) + hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 786e2e7e8dc..ab94ba8a73a 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -3,9 +3,17 @@ module Gitlab class OperationService include Gitlab::Git::Popen - WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do + BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created + + def self.from_gitaly(branch_update) + new( + branch_update.commit_id, + branch_update.repo_created, + branch_update.branch_created + ) + end end attr_reader :user, :repository @@ -112,7 +120,7 @@ module Gitlab ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name update_ref_in_hooks(ref, newrev, oldrev) - WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) + BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) end def find_oldrev_from_branch(newrev, branch) @@ -152,13 +160,15 @@ module Gitlab # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - _, status = popen( + + output, status = popen( command, repository.path) do |stdin| stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") end unless status.zero? + Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") raise Gitlab::Git::CommitError.new( "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ " Please refresh and try again.") diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 3d2fc471d28..b45da6020ee 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -5,6 +5,8 @@ require 'open3' module Gitlab module Git module Popen + FAST_GIT_PROCESS_TIMEOUT = 15.seconds + def popen(cmd, path, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" @@ -27,6 +29,67 @@ module Gitlab [@cmd_output, @cmd_status] end + + def popen_with_timeout(cmd, timeout, path, vars = {}) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + path ||= Dir.pwd + vars['PWD'] = path + + unless File.directory?(path) + FileUtils.mkdir_p(path) + end + + rout, wout = IO.pipe + rerr, werr = IO.pipe + + pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) + + begin + status = process_wait_with_timeout(pid, timeout) + + # close write ends so we could read them + wout.close + werr.close + + cmd_output = rout.readlines.join + cmd_output << rerr.readlines.join # Copying the behaviour of `popen` which merges stderr into output + + [cmd_output, status.exitstatus] + rescue Timeout::Error => e + kill_process_group_for_pid(pid) + + raise e + ensure + wout.close unless wout.closed? + werr.close unless werr.closed? + + rout.close + rerr.close + end + end + + def process_wait_with_timeout(pid, timeout) + deadline = timeout.seconds.from_now + wait_time = 0.01 + + while deadline > Time.now + sleep(wait_time) + _, status = Process.wait2(pid, Process::WNOHANG) + + return status unless status.nil? + end + + raise Timeout::Error, "Timeout waiting for process ##{pid}" + end + + def kill_process_group_for_pid(pid) + Process.kill("KILL", -pid) + Process.wait(pid) + rescue Errno::ESRCH + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 10ba29acbd1..59a54b48ed9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -12,6 +12,10 @@ module Gitlab GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES ].freeze + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze SEARCH_CONTEXT_LINES = 3 NoRepository = Class.new(StandardError) @@ -20,13 +24,11 @@ module Gitlab GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the storage path, not the storage name - def create(storage_path, name, bare: true, symlink_hooks_to: nil) - repo_path = File.join(storage_path, name) - repo_path += '.git' unless repo_path.end_with?('.git') - + # Unlike `new`, `create` takes the repository path + def create(repo_path, bare: true, symlink_hooks_to: nil) FileUtils.mkdir_p(repo_path, mode: 0770) # Equivalent to `git --git-path=#{repo_path} init [--bare]` @@ -55,14 +57,15 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver - # 'path' must be the path to a _bare_ git repository, e.g. - # /path/to/my-repo.git + # This initializer method is only used on the client side (gitlab-ce). + # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -73,8 +76,6 @@ module Gitlab delegate :empty?, to: :rugged - delegate :exists?, to: :gitaly_repository_client - def ==(other) path == other.path end @@ -102,6 +103,18 @@ module Gitlab @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end + def exists? + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + gitaly_repository_client.exists? + else + circuit_breaker.perform do + File.exist?(File.join(@path, 'refs')) + end + end + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -181,6 +194,28 @@ module Gitlab end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_repository_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -386,7 +421,13 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(self, c) } + gitaly_migrate(:find_commits) do |is_enabled| + if is_enabled + gitaly_commit_client.find_commits(options) + else + raw_log(options).map { |c| Commit.decorate(self, c) } + end + end end # Used in gitaly-ruby @@ -620,49 +661,60 @@ module Gitlab end def add_branch(branch_name, user:, target:) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + gitaly_migrate(:operation_user_create_branch) do |is_enabled| + if is_enabled + gitaly_add_branch(branch_name, user, target) + else + rugged_add_branch(branch_name, user, target) + end + end end def add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } + gitaly_migrate(:operation_user_add_tag) do |is_enabled| + if is_enabled + gitaly_add_tag(tag_name, user: user, target: target, message: message) + else + rugged_add_tag(tag_name, user: user, target: target, message: message) + end end - - OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex end def rm_branch(branch_name, user:) - OperationService.new(user, self).rm_branch(find_branch(branch_name)) + gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_delete_branch(branch_name, user) + else + OperationService.new(user, self).rm_branch(find_branch(branch_name)) + end + end end def rm_tag(tag_name, user:) - OperationService.new(user, self).rm_tag(find_tag(tag_name)) + gitaly_migrate(:operation_user_delete_tag) do |is_enabled| + if is_enabled + gitaly_operations_client.rm_tag(tag_name, user) + else + Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) + end + end end def find_tag(name) tags.find { |tag| tag.name == name } end - def merge(user, source_sha, target_branch, message) + def merge(user, source_sha, target_branch, message, &block) + gitaly_migrate(:operation_user_merge_branch) do |is_enabled| + if is_enabled + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) + else + rugged_merge(user, source_sha, target_branch, message, &block) + end + end + end + + def rugged_merge(user, source_sha, target_branch, message) committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) OperationService.new(user, self).with_branch(target_branch) do |start_commit| @@ -931,7 +983,11 @@ module Gitlab if start_repository == self yield commit(start_branch_name) else - sha = start_repository.commit(start_branch_name).sha + start_commit = start_repository.commit(start_branch_name) + + return yield nil unless start_commit + + sha = start_commit.sha if branch_commit = commit(sha) yield branch_commit @@ -946,9 +1002,9 @@ module Gitlab def with_repo_tmp_commit(start_repository, start_branch_name, sha) tmp_ref = fetch_ref( - start_repository.path, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - "refs/tmp/#{SecureRandom.hex}/head" + start_repository, + source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + target_ref: "refs/tmp/#{SecureRandom.hex}" ) yield commit(sha) @@ -960,8 +1016,9 @@ module Gitlab with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) + true else - raise Rugged::ReferenceError, 'source repository is empty' + false end end end @@ -979,13 +1036,27 @@ module Gitlab end end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) + def write_ref(ref_path, ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + + command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] + input = "update #{ref_path}\x00#{ref}\x00\x00" + output, status = circuit_breaker.perform do + popen(command, path) { |stdin| stdin.write(input) } + end + + raise GitError, output unless status.zero? end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - message, status = run_git(args) + def fetch_ref(source_repository, source_ref:, target_ref:) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| + if is_enabled + gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) + else + local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) + end + end # Make sure ref was created, and raise Rugged::ReferenceError when not raise Rugged::ReferenceError, message if status != 0 @@ -994,9 +1065,16 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args) + def run_git(args, env: {}) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path, env) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) end end @@ -1020,11 +1098,41 @@ module Gitlab # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? + end + + def fetch(remote = 'origin') + args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) + + popen(args, @path).last.zero? + end + + def blob_at(sha, path) + Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) + end + + def commit_index(user, branch_name, index, options) + committer = user_to_committer(user) + + OperationService.new(user, self).with_branch(branch_name) do + commit_params = options.merge( + tree: index.write_tree(rugged), + author: committer, + committer: committer + ) + + create_commit(commit_params) + end end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + end + + def gitaly_operations_client + @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) end def gitaly_ref_client @@ -1039,12 +1147,18 @@ module Gitlab @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) end + def gitaly_operation_client + @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e raise NoRepository.new(e) rescue GRPC::BadStatus => e raise CommandError.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) end private @@ -1151,7 +1265,16 @@ module Gitlab end def alternate_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact + relative_paths = Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + + if relative_paths.any? + relative_paths.map { |d| File.join(path, d) } + else + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES) + .flatten + .compact + .flat_map { |d| d.split(File::PATH_SEPARATOR) } + end end # Get the content of a blob for a given commit. If the blob is a commit @@ -1361,6 +1484,33 @@ module Gitlab false end + def gitaly_add_tag(tag_name, user:, target:, message: nil) + gitaly_operations_client.add_tag(tag_name, user, target, message) + end + + def rugged_add_tag(tag_name, user:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) + } + end + + Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + rescue Rugged::TagError + raise TagExistsError + end + def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1403,6 +1553,46 @@ module Gitlab file.write(gitattributes_content) end end + + def gitaly_add_branch(branch_name, user, target) + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex + end + + def rugged_add_branch(branch_name, user, target) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(user, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def local_fetch_ref(source_path, source_ref:, target_ref:) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + run_git(args) + end + + def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_resolver.address(source_repository.storage) + gitaly_token = gitaly_resolver.token(source_repository.storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + + run_git(args, env: env) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index e0943d3a3eb..60b2a4ec411 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -28,10 +28,10 @@ module Gitlab private def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" end output.split("\n") diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index ea634d39668..da74719ae87 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -1,24 +1,26 @@ module Gitlab module Git class User - attr_reader :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) end + # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628 def self.from_gitaly(gitaly_user) - new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end - def initialize(name, email, gl_id) + def initialize(username, name, email, gl_id) + @username = username @name = name @email = email @gl_id = gl_id end def ==(other) - [name, email, gl_id] == [other.name, other.email, other.gl_id] + [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb new file mode 100644 index 00000000000..d651c931a38 --- /dev/null +++ b/lib/gitlab/git/wiki.rb @@ -0,0 +1,115 @@ +module Gitlab + module Git + class Wiki + DuplicatePageError = Class.new(StandardError) + + CommitDetails = Struct.new(:name, :email, :message) do + def to_h + { name: name, email: email, message: message } + end + end + + def self.default_ref + 'master' + end + + # Initialize with a Gitlab::Git::Repository instance + def initialize(repository) + @repository = repository + end + + def repository_exists? + @repository.exists? + end + + def write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def delete_page(page_path, commit_details) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) + nil + end + + def update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + + def pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + + def page(title:, version: nil, dir: nil) + if version + version = Gitlab::Git::Commit.find(@repository, version).id + end + + gollum_page = gollum_wiki.page(title, version, dir) + return unless gollum_page + + new_page(gollum_page) + end + + def file(name, version) + version ||= self.class.default_ref + gollum_file = gollum_wiki.file(name, version) + return unless gollum_file + + Gitlab::Git::WikiFile.new(gollum_file) + end + + def page_versions(page_path) + current_page = gollum_page_by_path(page_path) + current_page.versions.map do |gollum_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) + new_version(gollum_page, gollum_git_commit.id) + end + end + + def preview_slug(title, format) + gollum_wiki.preview_page(title, '', format).url_path + end + + private + + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + + def gollum_page_by_path(page_path) + page_name = Gollum::Page.canonicalize_filename(page_path) + page_dir = File.split(page_path).first + + gollum_wiki.paged(page_name, page_dir) + end + + def new_page(gollum_page) + Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) + end + + def new_version(gollum_page, commit_id) + commit = Gitlab::Git::Commit.find(@repository, commit_id) + Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + end + + def assert_type!(object, klass) + unless object.is_a?(klass) + raise ArgumentError, "expected a #{klass}, got #{object.inspect}" + end + end + end + end +end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb new file mode 100644 index 00000000000..527f2a44dea --- /dev/null +++ b/lib/gitlab/git/wiki_file.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiFile + attr_reader :mime_type, :raw_data, :name + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::File is not serializable we must get all the data from + # 'gollum_file' during initialization, and NOT store it in an instance + # variable. + def initialize(gollum_file) + @mime_type = gollum_file.mime_type + @raw_data = gollum_file.raw_data + @name = gollum_file.name + end + end + end +end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb new file mode 100644 index 00000000000..a06bac4414f --- /dev/null +++ b/lib/gitlab/git/wiki_page.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + class WikiPage + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::Page is not serializable we must get all the data from + # 'gollum_page' during initialization, and NOT store it in an instance + # variable. + # + # Note that 'version' is a WikiPageVersion instance which it itself + # serializable. That means it's OK to store 'version' in an instance + # variable. + def initialize(gollum_page, version) + @url_path = gollum_page.url_path + @title = gollum_page.title + @format = gollum_page.format + @path = gollum_page.path + @raw_data = gollum_page.raw_data + @name = gollum_page.name + @historical = gollum_page.historical? + + @version = version + end + + def historical? + @historical + end + + def text_data + return @text_data if defined?(@text_data) + + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end + end + end +end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb new file mode 100644 index 00000000000..55f1afedcab --- /dev/null +++ b/lib/gitlab/git/wiki_page_version.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiPageVersion + attr_reader :commit, :format + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are + # serializable. + def initialize(commit, format) + @commit = commit + @format = format + end + + delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 62d1ecae676..42b59c106e2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -16,7 +16,9 @@ module Gitlab account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', - receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', + read_only: 'The repository is temporarily read-only. Please try again later.', + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -159,6 +161,14 @@ module Gitlab end def check_push_access!(changes) + if project.repository_read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + end + if deploy_key check_deploy_key_push_access! elsif user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1fe5155c093..98f1f45b338 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess ERROR_MESSAGES = { + read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze @@ -17,6 +18,10 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + true end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index a3c6b21a6a1..2e3e4fc3f1f 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -11,7 +11,7 @@ module Gitlab return false if ref_name.start_with?('refs/remotes/') Gitlab::Utils.system_silent( - %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name})) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cbd9ff406de..6c1ae19ff11 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -28,6 +28,7 @@ module Gitlab SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MAXIMUM_GITALY_CALLS = 30 + CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new private_constant :MUTEX @@ -69,17 +70,38 @@ module Gitlab # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. + # + # This method optionally takes a block which receives the keyword + # arguments hash 'kwargs' that will be passed to gRPC. This allows the + # caller to modify or augment the keyword arguments. The block must + # return a hash. + # + # For example: + # + # GitalyClient.call(storage, service, rpc, request) do |kwargs| + # kwargs.merge(deadline: Time.now + 10) + # end + # def self.call(storage, service, rpc, request) enforce_gitaly_request_limits(:call) - metadata = request_metadata(storage) - metadata = yield(metadata) if block_given? - stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend + kwargs = request_kwargs(storage) + kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend end - def self.request_metadata(storage) + def self.request_kwargs(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) - { metadata: { 'authorization' => "Bearer #{encoded_token}" } } + metadata = { + 'authorization' => "Bearer #{encoded_token}", + 'client_name' => CLIENT_NAME + } + + feature_stack = Thread.current[:gitaly_feature_stack] + feature = feature_stack && feature_stack[0] + metadata['call_site'] = feature.to_s if feature + + { metadata: metadata } end def self.token(storage) @@ -137,7 +159,14 @@ module Gitlab Gitlab::Metrics.measure(metric_name) do # Some migrate calls wrap other migrate calls allow_n_plus_1_calls do - yield is_enabled + feature_stack = Thread.current[:gitaly_feature_stack] ||= [] + feature_stack.unshift(feature) + begin + yield is_enabled + ensure + feature_stack.shift + Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? + end end end end @@ -151,7 +180,7 @@ module Gitlab actual_call_count = increment_call_count("gitaly_#{call_site}_actual") # Do no enforce limits in production - return if Rails.env.production? + return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"] # Check if this call is nested within a allow_n_plus_1_calls # block and skip check if it is @@ -228,10 +257,20 @@ module Gitlab path.read.chomp end + def self.timestamp(t) + Google::Protobuf::Timestamp.new(seconds: t.to_i) + end + def self.encode(s) + return "" if s.nil? + s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) + end + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index cf3a3554552..a2b50f2507e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -230,6 +230,26 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) end + def find_commits(options) + request = Gitaly::FindCommitsRequest.new( + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + disable_walk: options[:disable_walk] + ) + request.after = GitalyClient.timestamp(options[:after]) if options[:after] + request.before = GitalyClient.timestamp(options[:before]) if options[:before] + request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + + request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + + consume_commits_response(response) + end + private def call_commit_diff(request_params, options = {}) @@ -254,7 +274,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } } end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb new file mode 100644 index 00000000000..bd7c345ac01 --- /dev/null +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitalyClient + class NamespaceService + def initialize(storage) + @storage = storage + end + + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:namespace_exists, request).exists + end + + def add(name) + request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:add_namespace, request) + end + + def remove(name) + request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:remove_namespace, request) + end + + def rename(from, to) + request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) + + gitaly_client_call(:rename_namespace, request) + end + + private + + def gitaly_client_call(type, request) + GitalyClient.call(@storage, :namespace_service, type, request) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb new file mode 100644 index 00000000000..91f34011f6e --- /dev/null +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -0,0 +1,110 @@ +module Gitlab + module GitalyClient + class OperationService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def rm_tag(tag_name, user) + request = Gitaly::UserDeleteTagRequest.new( + repository: @gitaly_repo, + tag_name: GitalyClient.encode(tag_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def add_tag(tag_name, user, target, message) + request = Gitaly::UserCreateTagRequest.new( + repository: @gitaly_repo, + user: Util.gitaly_user(user), + tag_name: GitalyClient.encode(tag_name), + target_revision: GitalyClient.encode(target), + message: GitalyClient.encode(message.to_s) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + elsif response.exists + raise Gitlab::Git::Repository::TagExistsError + end + + Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Repository::InvalidRef, e + end + + def user_create_branch(branch_name, user, start_point) + request = Gitaly::UserCreateBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user), + start_point: GitalyClient.encode(start_point) + ) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_create_branch, request) + if response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + end + + branch = response.branch + return nil unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + end + + def user_delete_branch(branch_name, user) + request = Gitaly::UserDeleteBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def user_merge_branch(user, source_sha, target_branch, message) + request_enum = QueueEnumerator.new + response_enum = GitalyClient.call( + @repository.storage, + :operation_service, + :user_merge_branch, + request_enum.each + ) + + request_enum.push( + Gitaly::UserMergeBranchRequest.new( + repository: @gitaly_repo, + user: Util.gitaly_user(user), + commit_id: source_sha, + branch: GitalyClient.encode(target_branch), + message: GitalyClient.encode(message) + ) + ) + + yield response_enum.next.commit_id + + request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) + + branch_update = response_enum.next.branch_update + raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + ensure + request_enum.close + end + end + end +end diff --git a/lib/gitlab/gitaly_client/queue_enumerator.rb b/lib/gitlab/gitaly_client/queue_enumerator.rb new file mode 100644 index 00000000000..b8018029552 --- /dev/null +++ b/lib/gitlab/gitaly_client/queue_enumerator.rb @@ -0,0 +1,28 @@ +module Gitlab + module GitalyClient + class QueueEnumerator + def initialize + @queue = Queue.new + end + + def push(elem) + @queue << elem + end + + def close + push(:close) + end + + def each + return enum_for(:each) unless block_given? + + loop do + elem = @queue.pop + break if elem == :close + + yield elem + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8ef873d5848..b0c73395cb1 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -155,19 +155,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map do |gitaly_tag| - if gitaly_tag.target_commit.present? - gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - @repository, - encode!(gitaly_tag.name.dup), - gitaly_tag.id, - gitaly_commit, - encode!(gitaly_tag.message.chomp) - ) - end + message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 177a1284f38..cef692d3c2a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -53,6 +53,18 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request) end + + def create_repository + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :create_repository, request) + end + + def has_local_branches? + request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request) + + response.value + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 8fc937496af..a1222a7e718 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -2,12 +2,43 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path) + def repository(repository_storage, relative_path, gl_repository) + git_object_directory = Gitlab::Git::Env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence || + Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].presence + git_alternate_object_directories = + Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']).presence || + Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']).flat_map { |d| d.split(File::PATH_SEPARATOR) } + Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, - git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) + gl_repository: gl_repository, + git_object_directory: git_object_directory.to_s, + git_alternate_object_directories: git_alternate_object_directories + ) + end + + def gitaly_user(gitlab_user) + return unless gitlab_user + + Gitaly::User.new( + gl_id: Gitlab::GlId.gl_id(gitlab_user), + name: GitalyClient.encode(gitlab_user.name), + email: GitalyClient.encode(gitlab_user.email) + ) + end + + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) + if gitaly_tag.target_commit.present? + commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) + end + + Gitlab::Git::Tag.new( + repository, + Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), + gitaly_tag.id, + commit, + Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) ) end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index e21922070c1..8911b81ec9a 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -38,7 +38,7 @@ module Gitlab end def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 0d5039ddf5f..413872d7e08 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -34,6 +34,21 @@ module Gitlab end end + def subkeys_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + raw_keys = GPGME::Key.find(:public, fingerprints) + + raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| + primary_subkey_id = raw_key.primary_subkey.keyid + + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + { keyid: s.keyid, fingerprint: s.fingerprint } + end + end + end + end + def user_infos_from_key(key) using_tmp_keychain do fingerprints = CurrentKeyChain.fingerprints_from_key(key) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 86bd9f5b125..0f4ba6f83fc 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -43,7 +43,9 @@ module Gitlab # key belonging to the keyid. # This way we can add the key to the temporary keychain and extract # the proper signature. - gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + # NOTE: the invoked method is #fingerprint but it's only returning + # 16 characters (the format used by keyid) instead of 40. + gpg_key = find_gpg_key(verified_signature.fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) @@ -74,7 +76,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -98,6 +100,10 @@ module Gitlab def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + + def find_gpg_key(keyid) + GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + end end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index e085eab26c9..1991911ef6a 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,8 +9,8 @@ module Gitlab GpgSignature .select(:id, :commit_sha, :project_id) .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) - .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each { |sig| sig.gpg_commit.update_signature!(sig) } + .where(gpg_key_primary_keyid: @gpg_key.keyids) + .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end end end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb new file mode 100644 index 00000000000..4febb0ab430 --- /dev/null +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -0,0 +1,56 @@ +module Gitlab + module HookData + class IssuableBuilder + CHANGES_KEYS = %i[previous current].freeze + + attr_accessor :issuable + + def initialize(issuable) + @issuable = issuable + end + + def build(user: nil, changes: {}) + hook_data = { + object_kind: issuable.class.name.underscore, + user: user.hook_attrs, + project: issuable.project.hook_attrs, + object_attributes: issuable.hook_attrs, + labels: issuable.labels.map(&:hook_attrs), + changes: final_changes(changes.slice(*safe_keys)), + # DEPRECATED + repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) + } + + if issuable.is_a?(Issue) + hook_data[:assignees] = issuable.assignees.map(&:hook_attrs) if issuable.assignees.any? + else + hook_data[:assignee] = issuable.assignee.hook_attrs if issuable.assignee + end + + hook_data + end + + def safe_keys + issuable_builder::SAFE_HOOK_ATTRIBUTES + issuable_builder::SAFE_HOOK_RELATIONS + end + + private + + def issuable_builder + case issuable + when Issue + Gitlab::HookData::IssueBuilder + when MergeRequest + Gitlab::HookData::MergeRequestBuilder + end + end + + def final_changes(changes_hash) + changes_hash.reduce({}) do |hash, (key, changes_array)| + hash[key] = Hash[CHANGES_KEYS.zip(changes_array)] + hash + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb new file mode 100644 index 00000000000..de9cab80a02 --- /dev/null +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -0,0 +1,55 @@ +module Gitlab + module HookData + class IssueBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + branch_name + closed_at + confidential + created_at + deleted_at + description + due_date + id + iid + last_edited_at + last_edited_by_id + milestone_id + moved_to_id + project_id + relative_position + state + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignees + labels + ].freeze + + attr_accessor :issue + + def initialize(issue) + @issue = issue + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(issue), + total_time_spent: issue.total_time_spent, + human_total_time_spent: issue.human_total_time_spent, + human_time_estimate: issue.human_time_estimate, + assignee_ids: issue.assignee_ids, + assignee_id: issue.assignee_ids.first # This key is deprecated + } + + issue.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb new file mode 100644 index 00000000000..eaef19c9d04 --- /dev/null +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -0,0 +1,62 @@ +module Gitlab + module HookData + class MergeRequestBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + created_at + deleted_at + description + head_pipeline_id + id + iid + last_edited_at + last_edited_by_id + merge_commit_sha + merge_error + merge_params + merge_status + merge_user_id + merge_when_pipeline_succeeds + milestone_id + ref_fetched + source_branch + source_project_id + state + target_branch + target_project_id + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignee + labels + ].freeze + + attr_accessor :merge_request + + def initialize(merge_request) + @merge_request = merge_request + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(merge_request), + source: merge_request.source_project.try(:hook_attrs), + target: merge_request.target_project.hook_attrs, + last_commit: merge_request.diff_head_commit&.hook_attrs, + work_in_progress: merge_request.work_in_progress?, + total_time_spent: merge_request.total_time_spent, + human_total_time_spent: merge_request.human_total_time_spent, + human_time_estimate: merge_request.human_time_estimate + } + + merge_request.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2171c6c7bbb..dec8b4c5acd 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -53,6 +53,7 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules + - :cluster - :services - :hooks - protected_branches: diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3bc095a99a9..639f4f0c3f0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class ProjectTreeRestorer # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestones).freeze + RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 380b336395d..469b230377d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,6 +8,8 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + cluster: 'Gcp::Cluster', + clusters: 'Gcp::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -35,7 +37,7 @@ module Gitlab def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id) + @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @user = user @project = project @@ -56,22 +58,21 @@ module Gitlab private def setup_models - if @relation_name == :notes - set_note_author - - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil + case @relation_name + when :merge_request_diff then setup_st_diff_commits + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :project_label, :project_labels then setup_label + when :milestone, :milestones then setup_milestone + else + @relation_hash['project_id'] = @project.id end update_user_references update_project_references - handle_group_label if group_label? reset_tokens! remove_encrypted_attributes! - - set_st_diff_commits if @relation_name == :merge_request_diff - set_diff if @relation_name == :merge_request_diff_files end def update_user_references @@ -82,6 +83,12 @@ module Gitlab end end + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + # Sets the author for a note. If the user importing the project # has admin access, an actual mapping with new project members # will be used. Otherwise, a note stating the original author name @@ -134,11 +141,9 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def group_label? - @relation_hash['type'] == 'GroupLabel' - end + def setup_label + return unless @relation_hash['type'] == 'GroupLabel' - def handle_group_label # If there's no group, move the label to a project label if @relation_hash['group_id'] @relation_hash['project_id'] = nil @@ -148,6 +153,14 @@ module Gitlab end end + def setup_milestone + if @relation_hash['group_id'] + @relation_hash['group_id'] = @project.group.id + else + @relation_hash['project_id'] = @project.id + end + end + def reset_tokens! return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) @@ -196,14 +209,14 @@ module Gitlab relation_class: relation_class) end - def set_st_diff_commits + def setup_st_diff_commits @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end - def set_diff + def setup_diff @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end @@ -248,7 +261,13 @@ module Gitlab end def find_or_create_object! - finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_attributes = if @relation_name == :group_label + %w[title group_id] + elsif parsed_relation_hash['project_id'] + %w[title project_id] + else + %w[title group_id] + end finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index cdbdfa10d0e..da43bd0af4b 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -113,7 +113,7 @@ module Gitlab def kubeconfig_embed_ca_pem(config, ca_pem) cluster = config.dig(:clusters, 0, :cluster) - cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + cluster[:'certificate-authority-data'] = Base64.strict_encode64(ca_pem) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index fb68627dedf..e60ceba27c8 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,7 +16,7 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true else diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index cd7e4ca7b7e..0afaa2306b5 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -22,8 +22,8 @@ module Gitlab Gitlab::LDAP::Config.new(provider) end - def users(field, value, limit = nil) - options = user_options(field, value, limit) + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -72,20 +72,24 @@ module Gitlab private - def user_options(field, value, limit) - options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } + def user_options(fields, value, limit) + options = { + attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + base: config.base + } + options[:size] = limit if limit - if field.to_sym == :dn + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + options[:base] = value options[:scope] = Net::LDAP::SearchScope_BaseObject - options[:filter] = user_filter else - options[:base] = config.base - options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) end - options + options.merge(filter: user_filter(filter)) end def user_filter(filter = nil) diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 4fbc5fa5262..3123da17fd9 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -3,6 +3,10 @@ module Gitlab module LDAP class AuthHash < Gitlab::OAuth::AuthHash + def uid + Gitlab::LDAP::Person.normalize_dn(super) + end + private def get_info(key) diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb new file mode 100644 index 00000000000..d6142dc6549 --- /dev/null +++ b/lib/gitlab/ldap/dn.rb @@ -0,0 +1,301 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end +end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 4d6f8ac79de..38d7a9ba2f5 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -17,6 +17,12 @@ module Gitlab adapter.user('dn', dn) end + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + def self.disabled_via_active_directory?(dn, adapter) adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end @@ -30,6 +36,26 @@ module Gitlab ] end + def self.normalize_dn(dn) + ::Gitlab::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @@ -52,7 +78,9 @@ module Gitlab attribute_value(:email) end - delegate :dn, to: :entry + def dn + self.class.normalize_dn(entry.dn) + end private diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3bf27b37ae6..1793097363e 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -17,41 +17,19 @@ module Gitlab end end - def initialize(auth_hash) - super - update_user_attributes - end - def save super('LDAP') end # instance methods - def gl_user - @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user + def find_user + find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) - end - - def update_user_attributes - if persisted? - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) - - # For a new identity set extern_uid to the LDAP DN - # For an existing identity with matching email but changed DN, update the DN. - # For an existing identity with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid - end - end - def changed? gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb deleted file mode 100644 index 306923902e0..00000000000 --- a/lib/gitlab/markdown/pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Gitlab - module Markdown - class Pipeline - def self.[](name) - name ||= :full - const_get("#{name.to_s.camelize}Pipeline") - end - - def self.filters - [] - end - - def self.transform_context(context) - context - end - - def self.html_pipeline - @html_pipeline ||= HTML::Pipeline.new(filters) - end - - class << self - %i(call to_document to_html).each do |meth| - define_method(meth) do |text, context| - context = transform_context(context) - - html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb new file mode 100644 index 00000000000..0de0cddcce4 --- /dev/null +++ b/lib/gitlab/middleware/read_only.rb @@ -0,0 +1,88 @@ +module Gitlab + module Middleware + class ReadOnly + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + API_VERSIONS = (3..4) + + def initialize(app) + @app = app + @whitelisted = internal_routes + end + + def call(env) + @env = env + + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + error_message = 'You cannot do writing operations on a read-only GitLab instance' + + if json_request? + return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] + else + rack_flash.alert = error_message + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(env) + end + + private + + def internal_routes + API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } + end + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def logout_route + route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + request.path.end_with?('.git/git-upload-pack') + end + + def lfs_route + request.path.end_with?('/info/lfs/objects/batch') + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7704bf715e4..47c2a422387 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -13,6 +13,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash update_profile if sync_profile_from_provider? + add_or_update_user_identities end def persisted? @@ -32,7 +33,7 @@ module Gitlab block_after_save = needs_blocking? - Users::UpdateService.new(gl_user).execute! + Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block if block_after_save @@ -44,47 +45,56 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider + return @gl_user if defined?(@gl_user) - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end + @gl_user = find_user + end - if signup_enabled? - @user ||= build_new_user - end + def find_user + user = find_by_uid_and_provider - if external_provider? && @user - @user.external = true - end + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user - @user + user end protected - def find_or_create_ldap_user + def add_or_update_user_identities + return unless gl_user + + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user return unless ldap_person - # If a corresponding person exists with same uid in a LDAP server, - # check if the user already has a GitLab account. user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) if user - # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) - else - log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." - user = find_by_uid_and_provider - if user.nil? - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - user = build_new_user - end - log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." - user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) + return user end - user + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) end def auto_link_ldap_user? @@ -108,9 +118,9 @@ module Gitlab end def find_ldap_person(auth_hash, adapter) - by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) end def ldap_config @@ -152,7 +162,7 @@ module Gitlab end def build_new_user - user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) + user_params = user_attributes.merge(skip_confirmation: true) Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 732fbf68dad..ae136202f0c 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -1,9 +1,9 @@ module Gitlab class ProjectTemplate - attr_reader :title, :name + attr_reader :title, :name, :description, :preview - def initialize(name, title) - @name, @title = name, title + def initialize(name, title, description, preview) + @name, @title, @description, @preview = name, title, description, preview end alias_method :logo, :name @@ -25,9 +25,9 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails'), - ProjectTemplate.new('spring', 'Spring'), - ProjectTemplate.new('express', 'NodeJS Express') + ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, gemfile, rakefile, and .gitlab-ci.yml file, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), + ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw, pom.xml, and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), + ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') ].freeze class << self diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb new file mode 100644 index 00000000000..3f52402b31f --- /dev/null +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -0,0 +1,54 @@ +module Gitlab + module QuickActions + # This class takes spend command argument + # and separates date and time from spend command arguments if it present + # example: + # spend_command_time_and_date = "15m 2017-01-02" + # SpendTimeAndDateSeparator.new(spend_command_time_and_date).execute + # => [900, Mon, 02 Jan 2017] + # if date doesn't present return time with current date + # in other cases return nil + class SpendTimeAndDateSeparator + DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + + def initialize(spend_command_arg) + @spend_arg = spend_command_arg + end + + def execute + return if @spend_arg.blank? + return [get_time, DateTime.now.to_date] unless date_present? + return unless valid_date? + + [get_time, get_date] + end + + private + + def get_time + raw_time = @spend_arg.gsub(DATE_REGEX, '') + Gitlab::TimeTrackingFormatter.parse(raw_time) + end + + def get_date + string_date = @spend_arg.match(DATE_REGEX)[0] + Date.parse(string_date) + end + + def date_present? + DATE_REGEX =~ @spend_arg + end + + def valid_date? + string_date = @spend_arg.match(DATE_REGEX)[0] + date = Date.parse(string_date) rescue nil + + date_past_or_today?(date) + end + + def date_past_or_today?(date) + date&.past? || date&.today? + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 58f6245579a..bd677ec4bf3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -65,5 +65,9 @@ module Gitlab "can contain only lowercase letters, digits, and '-'. " \ "Must start with a letter, and cannot end with '-'" end + + def build_trace_section_regex + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 0f323a9e8b2..e0a9d1dee77 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -10,41 +10,20 @@ module Gitlab super('SAML') end - def gl_user - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end - - @user ||= find_by_uid_and_provider - - if auto_link_saml_user? - @user ||= find_by_email - end + def find_user + user = find_by_uid_and_provider - if signup_enabled? - @user ||= build_new_user - end + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? - if external_users_enabled? && @user + if external_users_enabled? && user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - @user.external = - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - false - else - true - end + user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? end - @user - end - - def find_by_email - if auth_hash.has_attribute?(:email) - user = ::User.find_by(email: auth_hash.email.downcase) - user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user - user - end + user end def changed? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 81ecdf43ef9..a37112ae5c4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -65,7 +65,7 @@ module Gitlab # Init new repository # - # storage - project's storage path + # storage - project's storage name # name - project path with namespace # # Ex. @@ -73,7 +73,19 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + relative_path = name.dup + relative_path << '.git' unless relative_path.end_with?('.git') + + gitaly_migrate(:create_repository) do |is_enabled| + if is_enabled + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + repository.gitaly_repository_client.create_repository + true + else + repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + end + end rescue => err Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false @@ -210,10 +222,18 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).add(name) + else + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + end + end rescue Errno::EEXIST => e Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Remove directory from repositories storage @@ -224,7 +244,15 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) - FileUtils.rm_r(full_path(storage, name), force: true) + Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).remove(name) + else + FileUtils.rm_r(full_path(storage, name), force: true) + end + end + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Move namespace directory inside repositories storage @@ -234,9 +262,17 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) - return false if exists?(storage, new_name) || !exists?(storage, old_name) + Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).rename(old_name, new_name) + else + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + end + end + rescue GRPC::InvalidArgument + false end def url_to_repo(path) @@ -260,7 +296,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - File.exist?(full_path(storage, dir_name)) + Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + if enabled + gitaly_namespace_client(storage).exists?(dir_name) + else + File.exist?(full_path(storage, dir_name)) + end + end end protected @@ -337,6 +379,14 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + def gitaly_namespace_client(storage_path) + storage, _value = Gitlab.config.repositories.storages.find do |storage, value| + value['path'] == storage_path + end + + Gitlab::GitalyClient::NamespaceService.new(storage) + end + def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 104280f520a..d7d24eeb37b 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,7 +25,7 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']} "\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index a0a2769cf9e..a1f689d94d9 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -51,6 +51,13 @@ module Gitlab self.num_running(job_ids).zero? end + # Returns true if the given job is running + # + # job_id - The Sidekiq job ID to check. + def self.running?(job_id) + num_running([job_id]) > 0 + end + # Returns the number of jobs that are running. # # job_ids - The Sidekiq job IDs to check. diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 222021e8802..f30c771837a 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -12,8 +12,9 @@ module Gitlab # # Project.where("id IN (#{sql})") class Union - def initialize(relations) + def initialize(relations, remove_duplicates: true) @relations = relations + @remove_duplicates = remove_duplicates end def to_sql @@ -25,7 +26,11 @@ module Gitlab @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end - fragments.join("\nUNION\n") + fragments.join("\n#{union_keyword}\n") + end + + def union_keyword + @remove_duplicates ? 'UNION' : 'UNION ALL' end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 4e1ec1402ea..1caa791c1be 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,9 @@ module Gitlab class UrlSanitizer + ALLOWED_SCHEMES = %w[http https ssh git].freeze + def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) + regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -11,9 +13,9 @@ module Gitlab def self.valid?(url) return false unless url.present? - Addressable::URI.parse(url.strip) + uri = Addressable::URI.parse(url.strip) - true + ALLOWED_SCHEMES.include?(uri.scheme) rescue Addressable::URI::InvalidURIError false end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6857038dba8..70a403652e7 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -48,6 +48,9 @@ module Gitlab deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, + gcp_clusters: ::Gcp::Cluster.count, + gcp_clusters_enabled: ::Gcp::Cluster.enabled.count, + gcp_clusters_disabled: ::Gcp::Cluster.disabled.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 17550cf9074..58d5b0da1c4 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,9 +22,9 @@ module Gitlab params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + GL_USERNAME: user&.username, RepoPath: repo_path } - server = { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) @@ -89,6 +89,13 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + params.merge!( + 'GitalyServer' => gitaly_server_hash(repository), + 'GitalyRepository' => repository.gitaly_repository.to_h + ) + end + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -96,11 +103,16 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawDiffRequest' => Gitaly::RawDiffRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -109,11 +121,16 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -209,6 +226,22 @@ module Gitlab token: Gitlab::GitalyClient.token(repository.project.repository_storage) } end + + def workhorse_diff_or_patch_hash(repository, diff_refs) + { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.base_sha, + 'ShaTo' => diff_refs.head_sha + } + end + + def gitaly_diff_or_patch_hash(repository, diff_refs) + { + repository: repository.gitaly_repository, + left_commit_id: diff_refs.base_sha, + right_commit_id: diff_refs.head_sha + } + end end end end diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb new file mode 100644 index 00000000000..99a82c849e0 --- /dev/null +++ b/lib/google_api/auth.rb @@ -0,0 +1,54 @@ +module GoogleApi + class Auth + attr_reader :access_token, :redirect_uri, :state + + ConfigMissingError = Class.new(StandardError) + + def initialize(access_token, redirect_uri, state: nil) + @access_token = access_token + @redirect_uri = redirect_uri + @state = state + end + + def authorize_url + client.auth_code.authorize_url( + redirect_uri: redirect_uri, + scope: scope, + state: state # This is used for arbitary redirection + ) + end + + def get_token(code) + ret = client.auth_code.get_token(code, redirect_uri: redirect_uri) + return ret.token, ret.expires_at + end + + protected + + def scope + raise NotImplementedError + end + + private + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "google_oauth2" } + end + + def client + return @client if defined?(@client) + + unless config + raise ConfigMissingError + end + + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + site: 'https://accounts.google.com', + token_url: '/o/oauth2/token', + authorize_url: '/o/oauth2/auth' + ) + end + end +end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb new file mode 100644 index 00000000000..a440a3e3562 --- /dev/null +++ b/lib/google_api/cloud_platform/client.rb @@ -0,0 +1,88 @@ +require 'google/apis/container_v1' + +module GoogleApi + module CloudPlatform + class Client < GoogleApi::Auth + DEFAULT_MACHINE_TYPE = 'n1-standard-1'.freeze + SCOPE = 'https://www.googleapis.com/auth/cloud-platform'.freeze + LEAST_TOKEN_LIFE_TIME = 10.minutes + + class << self + def session_key_for_token + :cloud_platform_access_token + end + + def session_key_for_expires_at + :cloud_platform_expires_at + end + + def new_session_key_for_redirect_uri + SecureRandom.hex.tap do |state| + yield session_key_for_redirect_uri(state) + end + end + + def session_key_for_redirect_uri(state) + "cloud_platform_second_redirect_uri_#{state}" + end + end + + def scope + SCOPE + end + + def validate_token(expires_at) + return false unless access_token + return false unless expires_at + + # Making sure that the token will have been still alive during the cluster creation. + return false if token_life_time(expires_at) < LEAST_TOKEN_LIFE_TIME + + true + end + + def projects_zones_clusters_get(project_id, zone, cluster_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_cluster(project_id, zone, cluster_id) + end + + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + request_body = Google::Apis::ContainerV1::CreateClusterRequest.new( + { + "cluster": { + "name": cluster_name, + "initial_node_count": cluster_size, + "node_config": { + "machine_type": machine_type + } + } + } ) + + service.create_cluster(project_id, zone, request_body) + end + + def projects_zones_operations(project_id, zone, operation_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_operation(project_id, zone, operation_id) + end + + def parse_operation_id(self_link) + m = self_link.match(%r{projects/.*/zones/.*/operations/(.*)}) + m[1] if m + end + + private + + def token_life_time(expires_at) + DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc + end + end + end +end diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb new file mode 100644 index 00000000000..a17ae55910e --- /dev/null +++ b/lib/rspec_flaky/config.rb @@ -0,0 +1,21 @@ +require 'json' + +module RspecFlaky + class Config + def self.generate_report? + ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + end + + def self.suite_flaky_examples_report_path + ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json") + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json") + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json") + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb index f81fb90e870..6be24014d89 100644 --- a/lib/rspec_flaky/flaky_example.rb +++ b/lib/rspec_flaky/flaky_example.rb @@ -9,24 +9,21 @@ module RspecFlaky line: example.line, description: example.description, last_attempts_count: example.attempts, - flaky_reports: 1) + flaky_reports: 0) else super end end - def first_flaky_at - self[:first_flaky_at] || Time.now - end - - def last_flaky_at - Time.now - end + def update_flakiness!(last_attempts_count: nil) + self.first_flaky_at ||= Time.now + self.last_flaky_at = Time.now + self.flaky_reports += 1 + self.last_attempts_count = last_attempts_count if last_attempts_count - def last_flaky_job - return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] - - "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end end def to_h diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..973c95b0212 --- /dev/null +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,37 @@ +require 'json' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def self.from_json(json) + new(JSON.parse(json)) + end + + def initialize(collection = {}) + unless collection.is_a?(Hash) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" + end + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_report + Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys + end + + def -(other) + unless other.respond_to?(:key) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" + end + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index ec2fbd9e36c..4a5bfec9967 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -2,11 +2,15 @@ require 'json' module RspecFlaky class Listener - attr_reader :all_flaky_examples, :new_flaky_examples - - def initialize - @new_flaky_examples = {} - @all_flaky_examples = init_all_flaky_examples + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end def example_passed(notification) @@ -14,29 +18,21 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example_hash = all_flaky_examples[current_example.uid] - - all_flaky_examples[current_example.uid] = - if flaky_example_hash - FlakyExample.new(flaky_example_hash).tap do |ex| - ex.last_attempts_count = current_example.attempts - ex.flaky_reports += 1 - end - else - FlakyExample.new(current_example).tap do |ex| - new_flaky_examples[current_example.uid] = ex - end - end + flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(all_flaky_examples, all_flaky_examples_report_path) + write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) - write_report_file(new_flaky_examples, new_flaky_examples_report_path) + write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end @@ -46,30 +42,23 @@ module RspecFlaky private - def init_all_flaky_examples - return {} unless File.exist?(all_flaky_examples_report_path) + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + unless suite_flaky_examples_json + return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + end - Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + FlakyExamplesCollection.from_json(suite_flaky_examples_json) end - def write_report_file(examples, file_path) - return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + def write_report_file(examples_collection, file_path) + return unless RspecFlaky::Config.generate_report? report_path_dir = File.dirname(file_path) FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - File.write(file_path, JSON.pretty_generate(to_report(examples))) - end - - def all_flaky_examples_report_path - @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/all-report.json") - end - def new_flaky_examples_report_path - @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/new-report.json") + File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 7b486d78cf0..9af21078403 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -5,15 +5,16 @@ module SystemCheck # whitelisted as it may change the SSH client's behaviour dramatically. WHITELIST = %w[ authorized_keys + authorized_keys.lock authorized_keys2 known_hosts ].freeze set_name 'Git user has default SSH configuration?' - set_skip_reason 'skipped (git user is not present or configured)' + set_skip_reason 'skipped (GitLab read-only, or git user is not present / configured)' def skip? - !home_dir || !File.directory?(home_dir) + Gitlab::Database.read_only? || !home_dir || !File.directory?(home_dir) end def check? diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index c388682dfb4..6ee8c8874ec 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%W(#{Gitlab.config.git.bin_path} --version))) end def check? diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index fd82f5f8a4a..08a2c495bd4 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version))) end def check? diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 259a755d724..a42f02a84fd 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -3,8 +3,8 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all frontend assets' task compile: [ 'yarn:check', - 'rake:assets:precompile', 'gettext:po_to_json', + 'rake:assets:precompile', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 3eade7bf553..b4d05f5995a 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -4,7 +4,10 @@ namespace :gitlab do task :ee_compat_check, [:branch] => :environment do |_, args| opts = if ENV['CI'] - { branch: ENV['CI_COMMIT_REF_NAME'] } + { + ce_repo: ENV['CI_REPOSITORY_URL'], + branch: ENV['CI_COMMIT_REF_NAME'] + } else unless args[:branch] puts "Must specify a branch as an argument".color(:red) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 08677a98fc1..8377fe3269d 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -50,6 +50,8 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -67,6 +69,11 @@ namespace :gitlab do storages << { name: key, path: val['path'] } end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 42825f29e32..0e6aed32c52 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -79,7 +79,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage_path, + if Gitlab::Shell.new.add_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 4d485108cf6..7f86fd7b45e 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -39,13 +39,19 @@ class GithubImport def import! @project.force_import_start + import_success = false + timings = Benchmark.measure do - Github::Import.new(@project, @options).execute + import_success = Github::Import.new(@project, @options).execute end - puts "Import finished. Timings: #{timings}".color(:green) - - @project.import_finish + if import_success + @project.import_finish + puts "Import finished. Timings: #{timings}".color(:green) + else + puts "Import was not successful. Errors were as follows:" + puts @project.import_error + end end def new_project @@ -53,18 +59,23 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Projects::CreateService.new( + project = Projects::CreateService.new( @current_user, name: name, path: name, description: @repo['description'], namespace_id: namespace.id, visibility_level: visibility_level, - import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"), skip_wiki: @repo['has_wiki'] ).execute + + project.update!( + import_type: 'github', + import_source: @repo['full_name'], + import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + ) + + project end end |