diff options
Diffstat (limited to 'lib')
173 files changed, 4045 insertions, 2040 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb new file mode 100644 index 00000000000..7b9de7c9598 --- /dev/null +++ b/lib/api/access_requests.rb @@ -0,0 +1,85 @@ +module API + class AccessRequests < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + resource source_type.pluralize do + # Get a list of group/project access requests viewable by the authenticated user. + # + # Parameters: + # id (required) - The group/project ID + # + # Example Request: + # GET /groups/:id/access_requests + # GET /projects/:id/access_requests + get ":id/access_requests" do + source = find_source(source_type, params[:id]) + + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) + + present access_requesters.map(&:user), with: Entities::AccessRequester, source: source + end + + # Request access to the group/project + # + # Parameters: + # id (required) - The group/project ID + # + # Example Request: + # POST /groups/:id/access_requests + # POST /projects/:id/access_requests + post ":id/access_requests" do + source = find_source(source_type, params[:id]) + access_requester = source.request_access(current_user) + + if access_requester.persisted? + present access_requester.user, with: Entities::AccessRequester, access_requester: access_requester + else + render_validation_error!(access_requester) + end + end + + # Approve a group/project access request + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the access requester + # access_level (optional) - Access level + # + # Example Request: + # PUT /groups/:id/access_requests/:user_id/approve + # PUT /projects/:id/access_requests/:user_id/approve + put ':id/access_requests/:user_id/approve' do + required_attributes! [:user_id] + source = find_source(source_type, params[:id]) + + member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute + + status :created + present member.user, with: Entities::Member, member: member + end + + # Deny a group/project access request + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the access requester + # + # Example Request: + # DELETE /groups/:id/access_requests/:user_id + # DELETE /projects/:id/access_requests/:user_id + delete ":id/access_requests/:user_id" do + required_attributes! [:user_id] + source = find_source(source_type, params[:id]) + + access_requester = source.requesters.find_by!(user_id: params[:user_id]) + + ::Members::DestroyService.new(access_requester, current_user).execute + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd4a853dbe..cb47ec8f33f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,10 @@ module API include APIGuard version 'v3', using: :path + rescue_from Gitlab::Access::AccessDeniedError do + rack_response({ 'message' => '403 Forbidden' }.to_json, 403) + end + rescue_from ActiveRecord::RecordNotFound do rack_response({ 'message' => '404 Not found' }.to_json, 404) end @@ -14,45 +18,44 @@ module API end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + handle_api_exception(exception) end format :json content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::SentryHelper helpers ::API::Helpers + # Keep in alphabetical order + mount ::API::AccessRequests mount ::API::AwardEmoji mount ::API::Branches + mount ::API::BroadcastMessages mount ::API::Builds mount ::API::CommitStatuses mount ::API::Commits mount ::API::DeployKeys + mount ::API::Deployments mount ::API::Environments mount ::API::Files - mount ::API::GroupMembers mount ::API::Groups mount ::API::Internal mount ::API::Issues mount ::API::Keys mount ::API::Labels mount ::API::LicenseTemplates + mount ::API::Lint + mount ::API::Members mount ::API::MergeRequests + mount ::API::MergeRequestDiffs mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes + mount ::API::NotificationSettings + mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectMembers mount ::API::ProjectSnippets mount ::API::Projects mount ::API::Repositories diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..8cc7a26f1fa 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -33,46 +33,29 @@ module API # # If the token is revoked, then it raises RevokedError. # - # If the token is not found (nil), then it raises TokenNotFoundError. + # If the token is not found (nil), then it returns nil # # Arguments: # # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end - end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + access_token = find_access_token + return nil unless access_token + + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end @@ -96,19 +79,6 @@ module API end module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end - end - private def install_error_responders(base) diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2efe7e3adf3..2461a783ea8 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,12 +1,12 @@ module API class AwardEmoji < Grape::API before { authenticate! } - AWARDABLES = [Issue, MergeRequest] + AWARDABLES = %w[issue merge_request snippet] resource :projects do AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.to_s.underscore.pluralize - awardable_id_string = "#{awardable_type.to_s.underscore}_id" + awardable_string = awardable_type.pluralize + awardable_id_string = "#{awardable_type}_id" [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" @@ -54,7 +54,7 @@ module API post endpoint do required_attributes! [:name] - not_found!('Award Emoji') unless can_read_awardable? + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -87,27 +87,36 @@ module API helpers do def can_read_awardable? - ability = "read_#{awardable.class.to_s.underscore}".to_sym + can?(current_user, read_ability(awardable), awardable) + end - can?(current_user, ability, awardable) + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) end def awardable @awardable ||= begin if params.include?(:note_id) - noteable.notes.find(params[:note_id]) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + elsif params.include?(:merge_request_id) + user_project.merge_requests.find(params[:merge_request_id]) else - noteable + user_project.snippets.find(params[:snippet_id]) end end end - def noteable - if params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) else - user_project.merge_requests.find(params[:merge_request_id]) + :"read_#{awardable.class.to_s.underscore}" end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index a77afe634f6..b615703df93 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -61,22 +61,27 @@ module API name: @branch.name } - unless developers_can_merge.nil? - protected_branch_params.merge!({ - merge_access_level_attributes: { - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - } - }) + # If `developers_can_merge` is switched off, _all_ `DEVELOPER` + # merge_access_levels need to be deleted. + if developers_can_merge == false + protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all end - unless developers_can_push.nil? - protected_branch_params.merge!({ - push_access_level_attributes: { - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - } - }) + # If `developers_can_push` is switched off, _all_ `DEVELOPER` + # push_access_levels need to be deleted. + if developers_can_push == false + protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all end + protected_branch_params.merge!( + merge_access_levels_attributes: [{ + access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + }], + push_access_levels_attributes: [{ + access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + }] + ) + if protected_branch service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) service.execute(protected_branch) diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb new file mode 100644 index 00000000000..fb2a4148011 --- /dev/null +++ b/lib/api/broadcast_messages.rb @@ -0,0 +1,99 @@ +module API + class BroadcastMessages < Grape::API + before { authenticate! } + before { authenticated_as_admin! } + + resource :broadcast_messages do + helpers do + def find_message + BroadcastMessage.find(params[:id]) + end + end + + desc 'Get all broadcast messages' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + optional :page, type: Integer, desc: 'Current page number' + optional :per_page, type: Integer, desc: 'Number of messages per page' + end + get do + messages = BroadcastMessage.all + + present paginate(messages), with: Entities::BroadcastMessage + end + + desc 'Create a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time', default: -> { Time.zone.now } + optional :ends_at, type: DateTime, desc: 'Ending time', default: -> { 1.hour.from_now } + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + post do + create_params = declared(params, include_missing: false).to_h + message = BroadcastMessage.create(create_params) + + if message.persisted? + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Get a specific broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + get ':id' do + message = find_message + + present message, with: Entities::BroadcastMessage + end + + desc 'Update a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + optional :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time' + optional :ends_at, type: DateTime, desc: 'Ending time' + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + put ':id' do + message = find_message + update_params = declared(params, include_missing: false).to_h + + if message.update(update_params) + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Delete a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + delete ':id' do + message = find_message + + present message.destroy, with: Entities::BroadcastMessage + end + end + end +end diff --git a/lib/api/builds.rb b/lib/api/builds.rb index be5a3484ec8..52bdbcae5a8 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -189,6 +189,27 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) end + + desc 'Trigger a manual build' do + success Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Build") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 4df6ca8333e..dfbdd597d29 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -37,7 +37,7 @@ module API # id (required) - The ID of a project # sha (required) - The commit hash # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, error or failure + # state (required) - The state of the status. Can be: pending, running, success, failed or canceled # target_url (optional) - The target URL to associate with this status # description (optional) - A short description of the status # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" @@ -46,7 +46,7 @@ module API post ':id/statuses/:sha' do authorize! :create_commit_status, user_project required_attributes! [:state] - attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + attrs = attributes_for_keys [:target_url, :description] commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -58,36 +58,38 @@ module API # the first found branch on that commit ref = params[:ref] - unless ref - branches = @project.repository.branch_names_contains(commit.sha) - not_found! 'References for commit' if branches.none? - ref = branches.first - end + ref ||= @project.repository.branch_names_contains(commit.sha).first + not_found! 'References for commit' unless ref - pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) + name = params[:name] || params[:context] || 'default' - name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) - status.update(attrs) + pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) - case params[:state].to_s - when 'running' - status.run - when 'success' - status.success - when 'failed' - status.drop - when 'canceled' - status.cancel - else - status.status = params[:state].to_s - end + status = GenericCommitStatus.running_or_pending.find_or_initialize_by( + project: @project, pipeline: pipeline, + user: current_user, name: name, ref: ref) + status.attributes = attrs + + begin + case params[:state].to_s + when 'pending' + status.enqueue! + when 'running' + status.enqueue + status.run! + when 'success' + status.success! + when 'failed' + status.drop! + when 'canceled' + status.cancel! + else + render_api_error!('invalid state', 400) + end - if status.save present status, with: Entities::CommitStatus - else - render_validation_error!(status) + rescue StateMachines::InvalidTransition => e + render_api_error!(e.message, 400) end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb new file mode 100644 index 00000000000..f782bcaf7e9 --- /dev/null +++ b/lib/api/deployments.rb @@ -0,0 +1,40 @@ +module API + # Deployments RESTfull API endpoints + class Deployments < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: Entities::Deployment + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: Entities::Deployment + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e5b00dc45a5..04437322ec1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -15,7 +15,7 @@ module API class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :location, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end class Identity < Grape::Entity @@ -48,7 +48,8 @@ module API class ProjectHook < Hook expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :issues_events, :merge_requests_events, :tag_push_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events expose :enable_ssl_verification end @@ -75,32 +76,57 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :container_registry_enabled + + # Expose old field names with the new permissions methods to keep API compatible + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) } + expose :created_at, :last_activity_at expose :shared_runners_enabled + expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end + expose :only_allow_merge_if_build_succeeds + expose :request_access_enabled end - class ProjectMember < UserBasic + class Member < UserBasic expose :access_level do |user, options| - options[:project].project_members.find_by(user_id: user.id).access_level + member = options[:member] || options[:source].members.find_by(user_id: user.id) + member.access_level + end + expose :expires_at do |user, options| + member = options[:member] || options[:source].members.find_by(user_id: user.id) + member.expires_at + end + end + + class AccessRequester < UserBasic + expose :requested_at do |user, options| + access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id) + access_requester.requested_at end end class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level + expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url + expose :request_access_enabled end class GroupDetail < Group @@ -108,12 +134,6 @@ module API expose :shared_projects, using: Entities::Project end - class GroupMember < UserBasic - expose :access_level do |user, options| - options[:group].group_members.find_by(user_id: user.id).access_level - end - end - class RepoBranch < Grape::Entity expose :name @@ -127,12 +147,14 @@ module API expose :developers_can_push do |repo_branch, options| project = options[:project] - project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.push_access_level.access_level == Gitlab::Access::DEVELOPER } + access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end expose :developers_can_merge do |repo_branch, options| project = options[:project] - project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.merge_access_level.access_level == Gitlab::Access::DEVELOPER } + access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end end @@ -168,6 +190,10 @@ module API # TODO (rspeicher): Deprecated; remove in 9.0 expose(:expires_at) { |snippet| nil } + + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end end class ProjectEntity < Grape::Entity @@ -197,6 +223,11 @@ module API expose :user_notes_count expose :upvotes, :downvotes expose :due_date + expose :confidential + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end end class ExternalIssue < Grape::Entity @@ -214,12 +245,18 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :diff_head_sha, as: :sha + expose :merge_commit_sha expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch + + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end end class MergeRequestChanges < MergeRequest @@ -228,6 +265,19 @@ module API end end + class MergeRequestDiff < Grape::Entity + expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha, + :created_at, :merge_request_id, :state, :real_size + end + + class MergeRequestDiffFull < MergeRequestDiff + expose :commits, using: Entities::RepoCommit + + expose :diffs, using: Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a + end + end + class SSHKey < Grape::Entity expose :id, :title, :key, :created_at end @@ -293,7 +343,7 @@ module API end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -325,24 +375,40 @@ module API expose :id, :path, :kind end - class Member < Grape::Entity + class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| if member.notification_setting - NotificationSetting.levels[member.notification_setting.level] + ::NotificationSetting.levels[member.notification_setting.level] end end end - class ProjectAccess < Member + class ProjectAccess < MemberAccess + end + + class GroupAccess < MemberAccess end - class GroupAccess < Member + class NotificationSetting < Grape::Entity + expose :level + expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do + ::NotificationSetting::EMAIL_EVENTS.each do |event| + expose event + end + end + end + + class GlobalNotificationSetting < NotificationSetting + expose :notification_email do |notification_setting, options| + notification_setting.user.notification_email + end end class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :push_events, :issues_events, :merge_requests_events + expose :tag_push_events, :note_events, :build_events, :pipeline_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -428,6 +494,8 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :koding_enabled + expose :koding_url end class Release < Grape::Entity @@ -479,6 +547,10 @@ module API expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -486,6 +558,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -496,10 +569,29 @@ module API expose :key, :value end - class Environment < Grape::Entity + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors + + expose :user, with: Entities::UserBasic + expose :created_at, :updated_at, :started_at, :finished_at, :committed_at + expose :duration + end + + class EnvironmentBasic < Grape::Entity expose :id, :name, :external_url end + class Environment < EnvironmentBasic + expose :project, using: Entities::Project + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: Entities::UserBasic + expose :environment, using: Entities::EnvironmentBasic + expose :deployable, using: Entities::Build + end + class RepoLicense < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular @@ -519,5 +611,10 @@ module API class Template < Grape::Entity expose :name, :content end + + class BroadcastMessage < Grape::Entity + expose :id, :message, :starts_at, :ends_at, :color, :font + expose :active?, as: :active + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index c1d86f313b0..96510e651a3 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -11,14 +11,16 @@ module API target_branch: attrs[:branch_name], commit_message: attrs[:commit_message], file_content: attrs[:content], - file_content_encoding: attrs[:encoding] + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] } end def commit_response(attrs) { file_path: attrs[:file_path], - branch_name: attrs[:branch_name], + branch_name: attrs[:branch_name] } end end @@ -96,7 +98,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -122,7 +124,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -149,7 +151,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name] result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb deleted file mode 100644 index dbe5bb08d3f..00000000000 --- a/lib/api/group_members.rb +++ /dev/null @@ -1,87 +0,0 @@ -module API - class GroupMembers < Grape::API - before { authenticate! } - - resource :groups do - # Get a list of group members viewable by the authenticated user. - # - # Example Request: - # GET /groups/:id/members - get ":id/members" do - group = find_group(params[:id]) - users = group.users - present users, with: Entities::GroupMember, group: group - end - - # Add a user to the list of group members - # - # Parameters: - # id (required) - group id - # user_id (required) - the users id - # access_level (required) - Project access level - # Example Request: - # POST /groups/:id/members - post ":id/members" do - group = find_group(params[:id]) - authorize! :admin_group, group - required_attributes! [:user_id, :access_level] - - unless validate_access_level?(params[:access_level]) - render_api_error!("Wrong access level", 422) - end - - if group.group_members.find_by(user_id: params[:user_id]) - render_api_error!("Already exists", 409) - end - - group.add_users([params[:user_id]], params[:access_level], current_user) - member = group.group_members.find_by(user_id: params[:user_id]) - present member.user, with: Entities::GroupMember, group: group - end - - # Update group member - # - # Parameters: - # id (required) - The ID of a group - # user_id (required) - The ID of a group member - # access_level (required) - Project access level - # Example Request: - # PUT /groups/:id/members/:user_id - put ':id/members/:user_id' do - group = find_group(params[:id]) - authorize! :admin_group, group - required_attributes! [:access_level] - - group_member = group.group_members.find_by(user_id: params[:user_id]) - not_found!('User can not be found') if group_member.nil? - - if group_member.update_attributes(access_level: params[:access_level]) - @member = group_member.user - present @member, with: Entities::GroupMember, group: group - else - handle_member_errors group_member.errors - end - end - - # Remove member. - # - # Parameters: - # id (required) - group id - # user_id (required) - the users id - # - # Example Request: - # DELETE /groups/:id/members/:user_id - delete ":id/members/:user_id" do - group = find_group(params[:id]) - authorize! :admin_group, group - member = group.group_members.find_by(user_id: params[:user_id]) - - if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) - else - member.destroy - end - end - end - end -end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9d8b8d737a9..953fa474e88 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,17 +23,19 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # POST /groups post do - authorize! :create_group, current_user + authorize! :create_group required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] @group = Group.new(attrs) if @group.save @@ -47,17 +49,19 @@ module API # Update group. Available only for users who can administrate groups. # # Parameters: - # id (required) - The ID of a group - # path (optional) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # PUT /groups/:id put ':id' do group = find_group(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] if ::Groups::UpdateService.new(group, current_user, attrs).execute present group, with: Entities::GroupDetail @@ -97,7 +101,7 @@ module API group = find_group(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects - present projects, with: Entities::Project + present projects, with: Entities::Project, user: current_user end # Transfer a project to the Group namespace diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 130509cdad6..714d4ea3dc6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -12,13 +12,30 @@ module API nil end + def private_token + params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + def find_user_from_warden + warden ? warden.authenticate : nil + end + def find_user_by_private_token - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + token = private_token + return nil unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) end def current_user - @current_user ||= (find_user_by_private_token || doorkeeper_guard) + @current_user ||= find_user_by_private_token + @current_user ||= doorkeeper_guard + @current_user ||= find_user_from_warden unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil @@ -28,7 +45,7 @@ module API # If the sudo is the current user do nothing if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? + forbidden!('Must be admin to use sudo') unless @current_user.is_admin? @current_user = User.by_username_or_id(identifier) not_found!("No user id or username for: #{identifier}") if @current_user.nil? end @@ -49,16 +66,15 @@ module API def user_project @project ||= find_project(params[:id]) - @project || not_found!("Project") end def find_project(id) project = Project.find_with_namespace(id) || Project.find_by(id: id) - if project && can?(current_user, :read_project, project) + if can?(current_user, :read_project, project) project else - nil + not_found!('Project') end end @@ -89,11 +105,7 @@ module API end def find_group(id) - begin - group = Group.find(id) - rescue ActiveRecord::RecordNotFound - group = Group.find_by!(path: id) - end + group = Group.find_by(path: id) || Group.find_by(id: id) if can?(current_user, :read_group, group) group @@ -134,8 +146,8 @@ module API forbidden! unless current_user.is_admin? end - def authorize!(action, subject) - forbidden! unless abilities.allowed?(current_user, action, subject) + def authorize!(action, subject = nil) + forbidden! unless can?(current_user, action, subject) end def authorize_push_project @@ -153,7 +165,7 @@ module API end def can?(object, action, subject) - abilities.allowed?(object, action, subject) + Ability.allowed?(object, action, subject) end # Checks the occurrences of required attributes, each attribute must be present in the params hash @@ -197,10 +209,6 @@ module API errors end - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. # @@ -278,6 +286,10 @@ module API render_api_error!('304 Not Modified', 304) end + def no_content! + render_api_error!('204 No Content', 204) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -288,6 +300,24 @@ module API error!({ 'message' => message }, status) end + def handle_api_exception(exception) + if sentry_enabled? && report_exception?(exception) + define_params_for_grape_middleware + sentry_context + Raven.capture_exception(exception) + end + + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + end + # Projects helpers def filter_projects(projects) @@ -399,23 +429,10 @@ module API links.join(', ') end - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end - def secret_token File.read(Gitlab.config.gitlab_shell.secret_file).chomp end - def handle_member_errors(errors) - error!(errors[:access_level], 422) if errors[:access_level].any? - not_found!(errors) - end - def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' @@ -433,5 +450,19 @@ module API Entities::Issue end end + + # The Grape Error Middleware only has access to env but no params. We workaround this by + # defining a method that returns the right value. + def define_params_for_grape_middleware + self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + end + + # We could get a Grape or a standard Ruby exception. We should only report anything that + # is clearly an error. + def report_exception?(exception) + return true unless exception.respond_to?(:status) + + exception.status == 500 + end end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb new file mode 100644 index 00000000000..90114f6f667 --- /dev/null +++ b/lib/api/helpers/members_helpers.rb @@ -0,0 +1,13 @@ +module API + module Helpers + module MembersHelpers + def find_source(source_type, id) + public_send("find_#{source_type}", id) + end + + def authorize_admin_source!(source_type, source) + authorize! :"admin_#{source_type}", source + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 959b700de78..9a5d1ece070 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -35,6 +35,14 @@ module API Project.find_with_namespace(project_path) end end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end end post "/allowed" do @@ -51,9 +59,9 @@ module API access = if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol) + Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol) + Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) end access_status = access.check(params[:action], params[:changes]) @@ -74,6 +82,23 @@ module API response end + post "/lfs_authenticate" do + status 200 + + key = Key.find(params[:key_id]) + token_handler = Gitlab::LfsToken.new(key) + + { + username: token_handler.actor_name, + lfs_token: token_handler.token, + repository_http_path: project.http_url_to_repo + } + end + + get "/merge_request_urls" do + ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + end + # # Discover user by ssh key # @@ -97,6 +122,35 @@ module API {} end end + + post '/two_factor_recovery_codes' do + status 200 + + key = Key.find_by(id: params[:key_id]) + + unless key + return { 'success' => false, 'message' => 'Could not find the given key' } + end + + if key.is_a?(DeployKey) + return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user + + unless user + return { success: false, message: 'Could not find a user for the given key' } + end + + unless user.two_factor_enabled? + return { success: false, message: 'Two-factor authentication is not enabled for this user' } + end + + codes = user.generate_otp_backup_codes! + user.save! + + { success: true, recovery_codes: codes } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c4d3134da6c..c9689e6f8ef 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,8 +3,6 @@ module API class Issues < Grape::API before { authenticate! } - helpers ::Gitlab::AkismetHelper - helpers do def filter_issues_state(issues, state) case state @@ -43,7 +41,8 @@ module API issues = current_user.issues.inc_notes_with_associations issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -75,7 +74,11 @@ module API params[:group_id] = group.id params[:milestone_title] = params.delete(:milestone) params[:label_name] = params.delete(:labels) - params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + if params[:order_by] || params[:sort] + # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example) + params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}" + end issues = IssuesFinder.new(current_user, params).execute @@ -115,7 +118,8 @@ module API issues = filter_issues_milestone(issues, params[:milestone]) end - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end @@ -142,12 +146,13 @@ module API # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # POST /projects/:id/issues post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -156,21 +161,19 @@ module API render_api_error!({ labels: errors }, 400) end - project = user_project + attrs[:labels] = params[:labels] if params[:labels] - issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? + + issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels].present? - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) @@ -190,12 +193,13 @@ module API # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # PUT /projects/:id/issues/:issue_id put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -204,17 +208,15 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? + issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels] && can?(current_user, :admin_issue, user_project) - issue.remove_labels - # Create and add labels to the new created issue - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) diff --git a/lib/api/lint.rb b/lib/api/lint.rb new file mode 100644 index 00000000000..ae43a4a3237 --- /dev/null +++ b/lib/api/lint.rb @@ -0,0 +1,21 @@ +module API + class Lint < Grape::API + namespace :ci do + desc 'Validation of .gitlab-ci.yml content' + params do + requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + end + post '/lint' do + error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + + status 200 + + if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + end + end + end +end diff --git a/lib/api/members.rb b/lib/api/members.rb new file mode 100644 index 00000000000..37f0a6512f4 --- /dev/null +++ b/lib/api/members.rb @@ -0,0 +1,158 @@ +module API + class Members < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + resource source_type.pluralize do + # Get a list of group/project members viewable by the authenticated user. + # + # Parameters: + # id (required) - The group/project ID + # query - Query string + # + # Example Request: + # GET /groups/:id/members + # GET /projects/:id/members + get ":id/members" do + source = find_source(source_type, params[:id]) + + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] + users = paginate(users) + + present users, with: Entities::Member, source: source + end + + # Get a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # + # Example Request: + # GET /groups/:id/members/:user_id + # GET /projects/:id/members/:user_id + get ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + members = source.members + member = members.find_by!(user_id: params[:user_id]) + + present member.user, with: Entities::Member, member: member + end + + # Add a new group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the new member + # access_level (required) - A valid access level + # expires_at (optional) - Date string in the format YEAR-MONTH-DAY + # + # Example Request: + # POST /groups/:id/members + # POST /projects/:id/members + post ":id/members" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + required_attributes! [:user_id, :access_level] + + access_requester = source.requesters.find_by(user_id: params[:user_id]) + if access_requester + # We pass current_user = access_requester so that the requester doesn't + # receive a "access denied" email + ::Members::DestroyService.new(access_requester, access_requester.user).execute + end + + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but 409 behavior should be used + # for both project and group members in 9.0! + conflict!('Member already exists') if source_type == 'group' && member + + unless member + source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + member = source.members.find_by(user_id: params[:user_id]) + end + + if member + present member.user, with: Entities::Member, member: member + else + # Since `source.add_user` doesn't return a member object, we have to + # build a new one and populate its errors in order to render them. + member = source.members.build(attributes_for_keys([:user_id, :access_level, :expires_at])) + member.valid? # populate the errors + + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + # Update a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # access_level (required) - A valid access level + # expires_at (optional) - Date string in the format YEAR-MONTH-DAY + # + # Example Request: + # PUT /groups/:id/members/:user_id + # PUT /projects/:id/members/:user_id + put ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + required_attributes! [:user_id, :access_level] + + member = source.members.find_by!(user_id: params[:user_id]) + attrs = attributes_for_keys [:access_level, :expires_at] + + if member.update_attributes(attrs) + present member.user, with: Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + # Remove a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # + # Example Request: + # DELETE /groups/:id/members/:user_id + # DELETE /projects/:id/members/:user_id + delete ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + required_attributes! [:user_id] + + # This is to ensure back-compatibility but find_by! should be used + # in that casse in 9.0! + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but this should be removed in + # favor of find_by! in 9.0! + not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? + + # This is to ensure back-compatibility but 204 behavior should be used + # for all DELETE endpoints in 9.0! + if member.nil? + { message: "Access revoked", id: params[:user_id].to_i } + else + ::Members::DestroyService.new(member, current_user).execute + + present member.user, with: Entities::Member, member: member + end + end + end + end + end +end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb new file mode 100644 index 00000000000..07435d78468 --- /dev/null +++ b/lib/api/merge_request_diffs.rb @@ -0,0 +1,45 @@ +module API + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + before { authenticate! } + + resource :projects do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiff + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiffFull + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull + end + end + end +end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..9b73f6826cf 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -108,8 +108,7 @@ module API finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: @milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8bfa998dc53..c5c214d4d13 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -83,12 +83,12 @@ module API opts[:created_at] = params[:created_at] end - @note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::const_get(note.class.name) else - not_found!("Note #{@note.errors.messages}") + not_found!("Note #{note.errors.messages}") end end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb new file mode 100644 index 00000000000..a70a7e71073 --- /dev/null +++ b/lib/api/notification_settings.rb @@ -0,0 +1,97 @@ +module API + # notification_settings API + class NotificationSettings < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + resource :notification_settings do + desc 'Get global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + get do + notification_setting = current_user.global_notification_setting + + present notification_setting, with: Entities::GlobalNotificationSetting + end + + desc 'Update global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + params do + optional :level, type: String, desc: 'The global notification level' + optional :notification_email, type: String, desc: 'The email address to send notifications' + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put do + notification_setting = current_user.global_notification_setting + + begin + notification_setting.transaction do + new_notification_email = params.delete(:notification_email) + declared_params = declared(params, include_missing: false).to_h + + current_user.update(notification_email: new_notification_email) if new_notification_email + notification_setting.update(declared_params) + end + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! current_user + render_validation_error! notification_setting + present notification_setting, with: Entities::GlobalNotificationSetting + end + end + + %w[group project].each do |source_type| + resource source_type.pluralize do + desc "Get #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + end + get ":id/notification_settings" do + source = find_source(source_type, params[:id]) + + notification_setting = current_user.notification_settings_for(source) + + present notification_setting, with: Entities::NotificationSetting + end + + desc "Update #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + optional :level, type: String, desc: "The #{source_type} notification level" + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put ":id/notification_settings" do + source = find_source(source_type, params.delete(:id)) + notification_setting = current_user.notification_settings_for(source) + + begin + declared_params = declared(params, include_missing: false).to_h + + notification_setting.update(declared_params) + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! notification_setting + present notification_setting, with: Entities::NotificationSetting + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb new file mode 100644 index 00000000000..2a0c8e1f2c0 --- /dev/null +++ b/lib/api/pipelines.rb @@ -0,0 +1,77 @@ +module API + class Pipelines < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + optional :scope, type: String, values: ['running', 'branches', 'tags'], + desc: 'Either running, branches, or tags' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: Entities::Pipeline + end + + desc 'Gets a specific pipeline for the project' do + detail 'This feature was introduced in GitLab 8.11' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + get ':id/pipelines/:pipeline_id' do + authorize! :read_pipeline, user_project + + present pipeline, with: Entities::Pipeline + end + + desc 'Retry failed builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/retry' do + authorize! :update_pipeline, user_project + + pipeline.retry_failed(current_user) + + present pipeline, with: Entities::Pipeline + end + + desc 'Cancel all builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/cancel' do + authorize! :update_pipeline, user_project + + pipeline.cancel_running + + status 200 + present pipeline.reload, with: Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end +end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 6bb70bc8bc3..14f5be3b5f6 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -45,6 +45,8 @@ module API :tag_push_events, :note_events, :build_events, + :pipeline_events, + :wiki_page_events, :enable_ssl_verification ] @hook = user_project.hooks.new(attrs) @@ -78,6 +80,8 @@ module API :tag_push_events, :note_events, :build_events, + :pipeline_events, + :wiki_page_events, :enable_ssl_verification ] diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb deleted file mode 100644 index 6a0b3e7d134..00000000000 --- a/lib/api/project_members.rb +++ /dev/null @@ -1,110 +0,0 @@ -module API - # Projects members API - class ProjectMembers < Grape::API - before { authenticate! } - - resource :projects do - # Get a project team members - # - # Parameters: - # id (required) - The ID of a project - # query - Query string - # Example Request: - # GET /projects/:id/members - get ":id/members" do - if params[:query].present? - @members = paginate user_project.users.where("username LIKE ?", "%#{params[:query]}%") - else - @members = paginate user_project.users - end - present @members, with: Entities::ProjectMember, project: user_project - end - - # Get a project team members - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a user - # Example Request: - # GET /projects/:id/members/:user_id - get ":id/members/:user_id" do - @member = user_project.users.find params[:user_id] - present @member, with: Entities::ProjectMember, project: user_project - end - - # Add a new project team member - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a user - # access_level (required) - Project access level - # Example Request: - # POST /projects/:id/members - post ":id/members" do - authorize! :admin_project, user_project - required_attributes! [:user_id, :access_level] - - # either the user is already a team member or a new one - project_member = user_project.project_member(params[:user_id]) - if project_member.nil? - project_member = user_project.project_members.new( - user_id: params[:user_id], - access_level: params[:access_level] - ) - end - - if project_member.save - @member = project_member.user - present @member, with: Entities::ProjectMember, project: user_project - else - handle_member_errors project_member.errors - end - end - - # Update project team member - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a team member - # access_level (required) - Project access level - # Example Request: - # PUT /projects/:id/members/:user_id - put ":id/members/:user_id" do - authorize! :admin_project, user_project - required_attributes! [:access_level] - - project_member = user_project.project_members.find_by(user_id: params[:user_id]) - not_found!("User can not be found") if project_member.nil? - - if project_member.update_attributes(access_level: params[:access_level]) - @member = project_member.user - present @member, with: Entities::ProjectMember, project: user_project - else - handle_member_errors project_member.errors - end - end - - # Remove a team member from project - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a team member - # Example Request: - # DELETE /projects/:id/members/:user_id - delete ":id/members/:user_id" do - project_member = user_project.project_members.find_by(user_id: params[:user_id]) - - unless current_user.can?(:admin_project, user_project) || - current_user.can?(:destroy_project_member, project_member) - forbidden! - end - - if project_member.nil? - { message: "Access revoked", id: params[:user_id].to_i } - else - project_member.destroy - end - end - end - end -end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8fed7db8803..680055c95eb 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -51,7 +51,7 @@ module API @projects = current_user.viewable_starred_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::Project + present @projects, with: Entities::Project, user: current_user end # Get all projects for admin user @@ -91,8 +91,8 @@ module API # Create new project # # Parameters: - # name (required) - name for new project - # description (optional) - short project description + # name (required) - name for new project + # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -100,30 +100,35 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default + # namespace_id (optional) - defaults to user namespace + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) - 0 by default # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects post do required_attributes! [:name] - attrs = attributes_for_keys [:name, - :path, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, :namespace_id, + :only_allow_merge_if_build_succeeds, + :path, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :import_url, - :public_builds] + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -140,10 +145,10 @@ module API # Create new project for a specified user. Only available to admin users. # # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default + # user_id (required) - The ID of a user + # name (required) - name for new project + # description (optional) - short project description + # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -151,28 +156,33 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 + # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects/user/:user_id post "user/:user_id" do authenticated_as_admin! user = User.find(params[:user_id]) - attrs = attributes_for_keys [:name, - :description, + attrs = attributes_for_keys [:builds_enabled, :default_branch, + :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :import_url, - :public_builds] + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -183,16 +193,32 @@ module API end end - # Fork new project for the current user. + # Fork new project for the current user or provided namespace. # # Parameters: # id (required) - The ID of a project + # namespace (optional) - The ID or name of the namespace that the project will be forked into. # Example Request # POST /projects/fork/:id post 'fork/:id' do + attrs = {} + namespace_id = params[:namespace] + + if namespace_id.present? + namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) + + unless namespace && can?(current_user, :create_projects, namespace) + not_found!('Target Namespace') + end + + attrs[:namespace] = namespace + end + @forked_project = ::Projects::ForkService.new(user_project, - current_user).execute + current_user, + attrs).execute + if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else @@ -218,23 +244,27 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project # public_builds (optional) + # lfs_enabled (optional) # Example Request # PUT /projects/:id put ':id' do - attrs = attributes_for_keys [:name, - :path, - :description, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :default_branch, + :description, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, + :path, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :public_builds] + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -323,7 +353,7 @@ module API # DELETE /projects/:id delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).pending_delete! + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute end # Mark this project as forked from another @@ -363,23 +393,24 @@ module API # Share project with group # # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group + # id (required) - The ID of a project + # group_id (required) - The ID of a group # group_access (required) - Level of permissions for sharing + # expires_at (optional) - Share expiration date # # Example Request: # POST /projects/:id/share post ":id/share" do authorize! :admin_project, user_project required_attributes! [:group_id, :group_access] + attrs = attributes_for_keys [:group_id, :group_access, :expires_at] unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(attrs) + if link.save present link, with: Entities::ProjectGroupLink else @@ -405,18 +436,9 @@ module API # Example Request: # GET /projects/search/:query get "/search/:query" do - ids = current_user.authorized_projects.map(&:id) - visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] - projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") - sort = params[:sort] == 'desc' ? 'desc' : 'asc' - - projects = case params["order_by"] - when 'id' then projects.order("id #{sort}") - when 'name' then projects.order("name #{sort}") - when 'created_at' then projects.order("created_at #{sort}") - when 'last_activity_at' then projects.order("last_activity_at #{sort}") - else projects - end + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(project_order_by => project_sort) present paginate(projects), with: Entities::Project end diff --git a/lib/api/session.rb b/lib/api/session.rb index 56c202f1294..55ec66a6d67 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -14,6 +14,7 @@ module API user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user + return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? present user, with: Entities::UserLogin end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 18408797756..b9e718147e1 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,21 +1,28 @@ module API class Templates < Grape::API - TEMPLATE_TYPES = { - gitignores: Gitlab::Template::Gitignore, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYml + GLOBAL_TEMPLATE_TYPES = { + gitignores: Gitlab::Template::GitignoreTemplate, + gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate }.freeze - TEMPLATE_TYPES.each do |template, klass| + helpers do + def render_response(template_type, template) + not_found!(template_type.to_s.singularize) unless template + present template, with: Entities::Template + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, klass| # Get the list of the available template # # Example Request: # GET /gitignores # GET /gitlab_ci_ymls - get template.to_s do + get template_type.to_s do present klass.all, with: Entities::TemplatesList end - # Get the text for a specific template + # Get the text for a specific template present in local filesystem # # Parameters: # name (required) - The name of a template @@ -23,13 +30,10 @@ module API # Example Request: # GET /gitignores/Elixir # GET /gitlab_ci_ymls/Ruby - get "#{template}/:name" do + get "#{template_type}/:name" do required_attributes! [:name] - new_template = klass.find(params[:name]) - not_found!(template.to_s.singularize) unless new_template - - present new_template, with: Entities::Template + render_response(template_type, new_template) end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 26c24c3baff..19df13d8aac 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -61,9 +61,9 @@ module API # delete ':id' do todo = current_user.todos.find(params[:id]) - todo.done + TodoService.new.mark_todos_as_done([todo], current_user) - present todo, with: Entities::Todo, current_user: current_user + present todo.reload, with: Entities::Todo, current_user: current_user end # Mark all todos as done @@ -73,9 +73,7 @@ module API # delete do todos = find_todos - todos.each(&:done) - - todos.length + TodoService.new.mark_todos_as_done(todos, current_user) end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 8a376d3c2a3..18c4cad09ae 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -60,6 +60,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Number of projects user can create # extern_uid - External authentication provider UID # provider - External provider @@ -74,7 +75,7 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external, :organization] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) @@ -111,6 +112,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Limit projects each user can create # bio - Bio # location - Location of the user @@ -122,7 +124,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external, :organization] user = User.find(params[:id]) not_found!('User') unless user @@ -327,7 +329,7 @@ module API # Example Request: # GET /user get do - present @current_user, with: Entities::UserLogin + present @current_user, with: Entities::UserFull end # Get currently authenticated user's keys diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 654b4d1c896..cedbb289f6a 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -27,7 +27,7 @@ module Backup def backup_existing_files_dir timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") - if File.exists?(app_files_dir) + if File.exist?(app_files_dir) FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 2ff3e3bdfb0..0dfffaf0bc6 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -114,7 +114,7 @@ module Backup tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") - unless File.exists?(tar_file) + unless File.exist?(tar_file) puts "The specified backup doesn't exist!" exit 1 end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 1f5917b8127..9fcd9a3f999 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -28,7 +28,7 @@ module Backup wiki = ProjectWiki.new(project) - if File.exists?(path_to_repo(wiki)) + if File.exist?(path_to_repo(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) @@ -49,13 +49,13 @@ module Backup def restore Gitlab.config.repositories.storages.each do |name, path| - next unless File.exists?(path) + next unless File.exist?(path) # Move repos dir to 'repositories.old' dir bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) FileUtils.mv(path, bk_repos_path) # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 2770) + FileUtils.mkdir_p(path, mode: 02770) end Project.find_each(batch_size: 1000) do |project| @@ -63,7 +63,7 @@ module Backup project.ensure_dir_exist - if File.exists?(path_to_bundle(project)) + if File.exist?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) else @@ -80,7 +80,7 @@ module Backup wiki = ProjectWiki.new(project) - if File.exists?(path_to_bundle(wiki)) + if File.exist?(path_to_bundle(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index d77a5e3ff09..affe34394c2 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.object_class_title - @object_title ||= object_class.name.titleize - end - # Public: Find references in text (like `!123` for merge requests) # # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| @@ -49,10 +45,6 @@ module Banzai self.class.object_sym end - def object_class_title - self.class.object_class_title - end - def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -72,7 +64,7 @@ module Banzai end end - def project_from_ref_cache(ref) + def project_from_ref_cached(ref) if RequestStore.active? cache = project_refs_cache @@ -154,7 +146,7 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_text: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cache(project_ref) + project = project_from_ref_cached(project_ref) if project && object = find_object_cached(project, id) title = object_link_title(object) @@ -198,7 +190,7 @@ module Banzai end def object_link_title(object) - "#{object_class_title}: #{object.title}" + object.title end def object_link_text(object, matches) @@ -251,11 +243,27 @@ module Banzai end end - # Returns the projects for the given paths. - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) Project.where_paths_in(paths).includes(:namespace) end + # Returns projects for the given paths. + def find_projects_for_paths(paths) + if RequestStore.active? + to_query = paths - project_refs_cache.keys + + unless to_query.empty? + projects_relation_for_paths(to_query).each do |project| + get_or_set_cache(project_refs_cache, project.path_with_namespace) { project } + end + end + + project_refs_cache.slice(*paths).values + else + projects_relation_for_paths(paths) + end + end + def current_project_path @current_project_path ||= project.path_with_namespace end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index bbb88c979cc..4358bf45549 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -35,7 +35,7 @@ module Banzai end def object_link_title(range) - range.reference_title + nil end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 2ce1816672b..a26dd09c25a 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -28,10 +28,6 @@ module Banzai only_path: context[:only_path]) end - def object_link_title(commit) - commit.link_title - end - def object_link_text_extras(object, matches) extras = super diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4042e9a4c25..54c5f9a71a4 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -66,7 +66,7 @@ module Banzai end end - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e258dc8e2bf..8f262ef3d8d 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -70,6 +70,11 @@ module Banzai def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end + + def object_link_title(object) + # use title of wrapped element instead + nil + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index ca686c87d97..58fff496d00 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -59,6 +59,10 @@ module Banzai html_safe end end + + def object_link_title(object) + nil + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index bf058241cda..2d221290f7e 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -52,7 +52,7 @@ module Banzai end def reference_class(type) - "gfm gfm-#{type}" + "gfm gfm-#{type} has-tooltip" end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index ca80aac5a08..2470362e019 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -43,55 +43,57 @@ module Banzai whitelist[:protocols].delete('a') # ...but then remove links with unsafe protocols - whitelist[:transformers].push(remove_unsafe_links) + whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(remove_rel) + whitelist[:transformers].push(self.class.remove_rel) # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(clean_spans) + whitelist[:transformers].push(self.class.clean_spans) whitelist end - def remove_unsafe_links - lambda do |env| - node = env[:node] + class << self + def remove_unsafe_links + lambda do |env| + node = env[:node] - return unless node.name == 'a' - return unless node.has_attribute?('href') + return unless node.name == 'a' + return unless node.has_attribute?('href') - begin - uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + begin + uri = Addressable::URI.parse(node['href']) + uri.scheme = uri.scheme.strip.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + rescue Addressable::URI::InvalidURIError + node.remove_attribute('href') + end end end - end - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end end end - end - def clean_spans - lambda do |env| - node = env[:node] + def clean_spans + lambda do |env| + node = env[:node] - return unless node.name == 'span' - return unless node.has_attribute?('class') + return unless node.name == 'span' + return unless node.has_attribute?('class') - unless has_ancestor?(node, 'pre') - node.remove_attribute('class') - end + unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } + node.remove_attribute('class') + end - { node_whitelist: [node] } + { node_whitelist: [node] } + end end end end diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..4efbcaf5c7f 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -10,19 +10,21 @@ module Banzai # task_list gem. # # See https://github.com/github/task_list/pull/60 - class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) + module ClassNamesFilter + def add_css_class(node, *new_class_names) if new_class_names.include?('task-list') # Don't add class to all lists return elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') + super(node.parent, 'task-list') end - add_css_class_without_fix(node, *new_class_names) + super(node, *new_class_names) end + end - alias_method_chain :add_css_class, :fix + class TaskListFilter < TaskList::Filter + prepend ClassNamesFilter end end end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 2e2c8da311e..e7a1ec8457d 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -31,6 +31,7 @@ module Banzai def apply_relative_link_rules! if @uri.relative? && @uri.path.present? link = ::File.join(@wiki_base_path, @uri.path) + link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 6cf218aaa0d..f5d110e987b 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -79,7 +79,11 @@ module Banzai def referenced_by(nodes) ids = unique_attribute_values(nodes, self.class.data_attribute) - references_relation.where(id: ids) + if ids.empty? + references_relation.none + else + references_relation.where(id: ids) + end end # Returns the ActiveRecord::Relation to use for querying references in the @@ -211,7 +215,7 @@ module Banzai end def can?(user, permission, subject) - Ability.abilities.allowed?(user, permission, subject) + Ability.allowed?(user, permission, subject) end def find_projects_for_hash_keys(hash) diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 17bb99a2ae5..a6b9beecded 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -9,22 +9,14 @@ module Ci end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }, 500) + handle_api_exception(exception) end content_type :txt, 'text/plain' content_type :json, 'application/json' format :json + helpers ::SentryHelper helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 260ac81f5fa..59f85416ee5 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,7 +12,7 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact + update_runner_last_contact(save: false) update_runner_info required_attributes! [:token] not_found! unless current_runner.active? @@ -20,9 +20,14 @@ module Ci build = Ci::RegisterBuildService.new.execute(current_runner) if build + Gitlab::Metrics.add_event(:build_found, + project: build.project.path_with_namespace) + present build, with: Entities::BuildDetails else - not_found! + Gitlab::Metrics.add_event(:build_not_found) + + build_not_found! end end @@ -42,6 +47,9 @@ module Ci build.update_attributes(trace: params[:trace]) if params[:trace] + Gitlab::Metrics.add_event(:update_build, + project: build.project.path_with_namespace) + case params[:state].to_s when 'success' build.success @@ -93,6 +101,7 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build @@ -105,7 +114,8 @@ module Ci end status 200 - { TempPath: ArtifactUploader.artifacts_upload_path } + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok end # Upload artifacts to build - Runners only diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..66c05773b68 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -15,6 +15,15 @@ module Ci expose :filename, :size end + class BuildOptions < Grape::Entity + expose :image + expose :services + expose :artifacts + expose :cache + expose :dependencies + expose :after_script + end + class Build < Grape::Entity expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 199d62d9b8a..23353c62885 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 60 + UPDATE_RUNNER_EVERY = 40 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -14,19 +14,37 @@ module Ci end def authenticate_build_token!(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - forbidden! unless token && build.valid_token?(token) + forbidden! unless build_token_valid?(build) end def runner_registration_token_valid? - params[:token] == current_application_settings.runners_registration_token + ActiveSupport::SecurityUtils.variable_size_secure_compare( + params[:token], + current_application_settings.runners_registration_token) + end + + def build_token_valid?(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + + # We require to also check `runners_token` to maintain compatibility with old version of runners + token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact + def update_runner_last_contact(save: true) # Use a random threshold to prevent beating DB updates + # it generates a distribution between: [40m, 80m] contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.update_attributes(contacted_at: Time.now) + current_runner.contacted_at = Time.now + current_runner.save if current_runner.changed? && save + end + end + + def build_not_found! + if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + no_content! + else + not_found! end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index a2e8bd22a52..0369e80312a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -55,29 +55,36 @@ module Ci { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - ## - # Refactoring note: - # - before script behaves differently than after script - # - after script returns an array of commands - # - before script should be a concatenated command - commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), + commands: job[:commands], tag_list: job[:tags] || [], - name: job[:name], + name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], + environment: job[:environment_name], yaml_variables: yaml_variables(name), options: { - image: job[:image] || @image, - services: job[:services] || @services, + image: job[:image], + services: job[:services], artifacts: job[:artifacts], - cache: job[:cache] || @cache, + cache: job[:cache], dependencies: job[:dependencies], - after_script: job[:after_script] || @after_script, + after_script: job[:after_script], + environment: job[:environment], }.compact } end + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Ci::GitlabCiYamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + private def initial_parsing diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb new file mode 100644 index 00000000000..997377abc55 --- /dev/null +++ b/lib/ci/mask_secret.rb @@ -0,0 +1,10 @@ +module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end +end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb new file mode 100644 index 00000000000..7b1533d0d32 --- /dev/null +++ b/lib/expand_variables.rb @@ -0,0 +1,17 @@ +module ExpandVariables + class << self + def expand(value, variables) + # Convert hash array to variables + if variables.is_a?(Array) + variables = variables.reduce({}) do |hash, variable| + hash[variable[:key]] = variable[:value] + hash + end + end + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables[$1 || $2] + end + end + end +end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 51e46da82cc..a4558d157c0 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -94,7 +94,7 @@ module ExtractsPath @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } @options = HashWithIndifferentAccess.new(@options) - @id = Addressable::URI.unescape(get_id) + @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository if @options[:extended_sha1].blank? @@ -119,6 +119,7 @@ module ExtractsPath private + # overriden in subclasses, do not remove def get_id id = params[:id] || params[:ref] id += "/" + params[:path] unless params[:path].blank? diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb deleted file mode 100644 index 207736b59db..00000000000 --- a/lib/gitlab/akismet_helper.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module AkismetHelper - def akismet_enabled? - current_application_settings.akismet_enabled - end - - def akismet_client - @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, - Gitlab.config.gitlab.url) - end - - def client_ip(env) - env['action_dispatch.remote_ip'].to_s - end - - def user_agent(env) - env['HTTP_USER_AGENT'] - end - - def check_for_spam?(project) - akismet_enabled? && project.public? - end - - def is_spam?(environment, user, text) - client = akismet_client - ip_address = client_ip(environment) - user_agent = user_agent(environment) - - params = { - type: 'comment', - text: text, - created_at: DateTime.now, - author: user.name, - author_email: user.email, - referrer: environment['HTTP_REFERER'], - } - - begin - is_spam, is_blatant = client.check(ip_address, user_agent, params) - is_spam || is_blatant - rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") - false - end - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index db1704af75e..aca5d0020cf 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,22 +1,22 @@ module Gitlab module Auth - Result = Struct.new(:user, :type) + class MissingPersonalTokenError < StandardError; end class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? - result = Result.new + result = + service_request_check(login, password, project) || + build_access_token_check(login, password) || + user_with_password_for_git(login, password) || + oauth_access_token_check(login, password) || + lfs_token_check(login, password) || + personal_access_token_check(login, password) || + Gitlab::Auth::Result.new - if valid_ci_request?(login, password, project) - result.type = :ci - elsif result.user = find_with_user_password(login, password) - result.type = :gitlab_or_ldap - elsif result.user = oauth_access_token_check(login, password) - result.type = :oauth - end + rate_limit!(ip, success: result.success?, login: login) - rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) result end @@ -58,30 +58,117 @@ module Gitlab private - def valid_ci_request?(login, password, project) + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) - return false unless project && matched_login.present? + return unless project && matched_login.present? underscored_service = matched_login['service'].underscore - if underscored_service == 'gitlab_ci' - project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) + if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. service = project.public_send("#{underscored_service}_service") - service && service.activated? && service.valid_token?(password) + if service && service.activated? && service.valid_token?(password) + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) + end end end + def user_with_password_for_git(login, password) + user = find_with_user_password(login, password) + return unless user + + raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + + Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) + end + def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) + if token && token.accessible? + user = User.find_by(id: token.resource_owner_id) + Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + end + end + end + + def personal_access_token_check(login, password) + if login && password + user = User.find_by_personal_access_token(password) + validation = User.by_login(login) + Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + end + end + + def lfs_token_check(login, password) + deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) + + actor = + if deploy_key_matches + DeployKey.find(deploy_key_matches[1]) + else + User.by_login(login) + end + + return unless actor + + token_handler = Gitlab::LfsToken.new(actor) + + authentication_abilities = + if token_handler.user? + full_authentication_abilities + else + read_authentication_abilities + end + + Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password) + end + + def build_access_token_check(login, password) + return unless login == 'gitlab-ci-token' + return unless password + + build = ::Ci::Build.running.find_by_token(password) + return unless build + return unless build.project.builds_enabled? + + if build.user + # If user is assigned to build, use restricted credentials of user + Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) + else + # Otherwise use generic CI credentials (backward compatibility) + Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities) end end + + public + + def build_authentication_abilities + [ + :read_project, + :build_download_code, + :build_read_container_image, + :build_create_container_image + ] + end + + def read_authentication_abilities + [ + :read_project, + :download_code, + :read_container_image + ] + end + + def full_authentication_abilities + read_authentication_abilities + [ + :push_code, + :create_container_image + ] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb new file mode 100644 index 00000000000..6be7f690676 --- /dev/null +++ b/lib/gitlab/auth/result.rb @@ -0,0 +1,21 @@ +module Gitlab + module Auth + Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + def ci?(for_project) + type == :ci && + project && + project == for_project + end + + def lfs_deploy_token?(for_project) + type == :lfs_deploy_token && + actor && + actor.projects.include?(for_project) + end + + def success? + actor.present? || type == :ci + end + end + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb deleted file mode 100644 index ab94abeda77..00000000000 --- a/lib/gitlab/backend/grack_auth.rb +++ /dev/null @@ -1,163 +0,0 @@ -module Grack - class AuthSpawner - def self.call(env) - # Avoid issues with instance variables in Grack::Auth persisting across - # requests by creating a new instance for each request. - Auth.new({}).call(env) - end - end - - class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env - - def call(env) - @env = env - @request = Rack::Request.new(env) - @auth = Request.new(env) - - @ci = false - - # Need this patch due to the rails mount - # Need this if under RELATIVE_URL_ROOT - unless Gitlab.config.gitlab.relative_url_root.empty? - # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') - else - @env['PATH_INFO'] = @request.path - end - - @env['SCRIPT_NAME'] = "" - - auth! - - lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call - return lfs_response unless lfs_response.nil? - - if @user.nil? && !@ci - unauthorized - else - render_not_found - end - end - - private - - def auth! - return unless @auth.provided? - - return bad_request unless @auth.basic? - - # Authentication with username and password - login, password = @auth.credentials - - # Allow authentication for GitLab CI service - # if valid token passed - if ci_request?(login, password) - @ci = true - return - end - - @user = authenticate_user(login, password) - end - - def ci_request?(login, password) - matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) - - if project && matched_login.present? - underscored_service = matched_login['s'].underscore - - if underscored_service == 'gitlab_ci' - return project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) - service_method = "#{underscored_service}_service" - service = project.send(service_method) - - return service && service.activated? && service.valid_token?(password) - end - end - - false - end - - def oauth_access_token_check(login, password) - if login == "oauth2" && git_cmd == 'git-upload-pack' && password.present? - token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) - end - end - - def authenticate_user(login, password) - user = Gitlab::Auth.find_with_user_password(login, password) - - unless user - user = oauth_access_token_check(login, password) - end - - # If the user authenticated successfully, we reset the auth failure count - # from Rack::Attack for that IP. A client may attempt to authenticate - # with a username and blank password first, and only after it receives - # a 401 error does it present a password. Resetting the count prevents - # false positives from occurring. - # - # Otherwise, we let Rack::Attack know there was a failed authentication - # attempt from this IP. This information is stored in the Rails cache - # (Redis) and will be used by the Rack::Attack middleware to decide - # whether to block requests from this IP. - config = Gitlab.config.rack_attack.git_basic_auth - - if config.enabled - if user - # A successful login will reset the auth failure count from this IP - Rack::Attack::Allow2Ban.reset(@request.ip, config) - else - banned = Rack::Attack::Allow2Ban.filter(@request.ip, config) do - # Unless the IP is whitelisted, return true so that Allow2Ban - # increments the counter (stored in Rails.cache) for the IP - if config.ip_whitelist.include?(@request.ip) - false - else - true - end - end - - if banned - Rails.logger.info "IP #{@request.ip} failed to login " \ - "as #{login} but has been temporarily banned from Git auth" - end - end - end - - user - end - - def git_cmd - if @request.get? - @request.params['service'] - elsif @request.post? - File.basename(@request.path) - else - nil - end - end - - def project - return @project if defined?(@project) - - @project = project_by_path(@request.path_info) - end - - def project_by_path(path) - if m = /^([\w\.\/-]+)\.git/.match(path).to_a - path_with_namespace = m.last - path_with_namespace.gsub!(/\.wiki$/, '') - - path_with_namespace[0] = '' if path_with_namespace.start_with?('/') - Project.find_with_namespace(path_with_namespace) - end - end - - def render_not_found - [404, { "Content-Type" => "text/plain" }, ["Not Found"]] - end - end -end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 839a4fa30d5..79eac66b364 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -6,7 +6,12 @@ module Gitlab KeyAdder = Struct.new(:io) do def add_key(id, key) - key.gsub!(/[[:space:]]+/, ' ').strip! + key = Gitlab::Shell.strip_key(key) + # Newline and tab are part of the 'protocol' used to transmit id+key to the other end + if key.include?("\t") || key.include?("\n") + raise Error.new("Invalid key: #{key.inspect}") + end + io.puts("#{id}\t#{key}") end end @@ -16,6 +21,10 @@ module Gitlab @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip end + + def strip_key(key) + key.split(/ /)[0, 2].join(' ') + end end # Init new repository @@ -107,7 +116,7 @@ module Gitlab # def add_key(key_id, key_content) Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, key_content]) + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -195,7 +204,7 @@ module Gitlab # Create (if necessary) and link the secret token file def generate_and_link_secret_token secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.exist? secret_file + unless File.size?(secret_file) # Generate a new token of 16 random hexadecimal characters and store it in secret_file. token = SecureRandom.hex(16) File.write(secret_file, token) diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb new file mode 100644 index 00000000000..909fa24fa90 --- /dev/null +++ b/lib/gitlab/badge/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Badge + class Base + def entity + raise NotImplementedError + end + + def status + raise NotImplementedError + end + + def metadata + raise NotImplementedError + end + + def template + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb deleted file mode 100644 index e5e9fab3f5c..00000000000 --- a/lib/gitlab/badge/build.rb +++ /dev/null @@ -1,46 +0,0 @@ -module Gitlab - module Badge - ## - # Build badge - # - class Build - include Gitlab::Application.routes.url_helpers - include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper - - def initialize(project, ref) - @project, @ref = project, ref - @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) - end - - def type - 'image/svg+xml' - end - - def data - File.read(@image[:path]) - end - - def to_s - @image[:name].sub(/\.svg$/, '') - end - - def to_html - link_to(image_tag(image_url, alt: 'build status'), link_url) - end - - def to_markdown - "[](#{link_url})" - end - - def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) - end - - def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) - end - end - end -end diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb new file mode 100644 index 00000000000..f87a7b7942e --- /dev/null +++ b/lib/gitlab/badge/build/metadata.rb @@ -0,0 +1,28 @@ +module Gitlab + module Badge + module Build + ## + # Class that describes build badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + end + + def title + 'build status' + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb new file mode 100644 index 00000000000..50aa45e5406 --- /dev/null +++ b/lib/gitlab/badge/build/status.rb @@ -0,0 +1,37 @@ +module Gitlab + module Badge + module Build + ## + # Build status badge + # + class Status < Badge::Base + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + + @sha = @project.commit(@ref).try(:sha) + end + + def entity + 'build' + end + + def status + @project.pipelines + .where(sha: @sha, ref: @ref) + .status || 'unknown' + end + + def metadata + @metadata ||= Build::Metadata.new(self) + end + + def template + @template ||= Build::Template.new(self) + end + end + end + end +end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb new file mode 100644 index 00000000000..2b95ddfcb53 --- /dev/null +++ b/lib/gitlab/badge/build/template.rb @@ -0,0 +1,47 @@ +module Gitlab + module Badge + module Build + ## + # Class that represents a build badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + success: '#4c1', + failed: '#e05d44', + running: '#dfb317', + pending: '#dfb317', + canceled: '#9f9f9f', + skipped: '#9f9f9f', + unknown: '#9f9f9f' + } + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + @entity.to_s + end + + def value_text + @status.to_s + end + + def key_width + 38 + end + + def value_width + 54 + end + + def value_color + STATUS_COLOR[@status.to_sym] || STATUS_COLOR[:unknown] + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb new file mode 100644 index 00000000000..53588185622 --- /dev/null +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -0,0 +1,30 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that describes coverage badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + @job = badge.job + end + + def title + 'coverage report' + end + + def image_url + coverage_namespace_project_badges_url(@project.namespace, + @project, @ref, + format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb new file mode 100644 index 00000000000..9a0482306b7 --- /dev/null +++ b/lib/gitlab/badge/coverage/report.rb @@ -0,0 +1,53 @@ +module Gitlab + module Badge + module Coverage + ## + # Test coverage report badge + # + class Report < Badge::Base + attr_reader :project, :ref, :job + + def initialize(project, ref, job = nil) + @project = project + @ref = ref + @job = job + + @pipeline = @project.pipelines.latest_successful_for(@ref) + end + + def entity + 'coverage' + end + + def status + @coverage ||= raw_coverage + return unless @coverage + + @coverage.to_i + end + + def metadata + @metadata ||= Coverage::Metadata.new(self) + end + + def template + @template ||= Coverage::Template.new(self) + end + + private + + def raw_coverage + return unless @pipeline + + if @job.blank? + @pipeline.coverage + else + @pipeline.builds + .find_by(name: @job) + .try(:coverage) + end + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb new file mode 100644 index 00000000000..06e0d084e9f --- /dev/null +++ b/lib/gitlab/badge/coverage/template.rb @@ -0,0 +1,52 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that represents a coverage badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + good: '#4c1', + acceptable: '#a3c51c', + medium: '#dfb317', + low: '#e05d44', + unknown: '#9f9f9f' + } + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + @entity.to_s + end + + def value_text + @status ? "#{@status}%" : 'unknown' + end + + def key_width + 62 + end + + def value_width + @status ? 36 : 58 + end + + def value_color + case @status + when 95..100 then STATUS_COLOR[:good] + when 90..95 then STATUS_COLOR[:acceptable] + when 75..90 then STATUS_COLOR[:medium] + when 0..75 then STATUS_COLOR[:low] + else + STATUS_COLOR[:unknown] + end + end + end + end + end +end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb new file mode 100644 index 00000000000..548f85b78bb --- /dev/null +++ b/lib/gitlab/badge/metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module Badge + ## + # Abstract class for badge metadata + # + class Metadata + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(badge) + @badge = badge + end + + def to_html + link_to(image_tag(image_url, alt: title), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def title + raise NotImplementedError + end + + def image_url + raise NotImplementedError + end + + def link_url + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb new file mode 100644 index 00000000000..bfeb0052642 --- /dev/null +++ b/lib/gitlab/badge/template.rb @@ -0,0 +1,49 @@ +module Gitlab + module Badge + ## + # Abstract template class for badges + # + class Template + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + raise NotImplementedError + end + + def value_text + raise NotImplementedError + end + + def key_width + raise NotImplementedError + end + + def value_width + raise NotImplementedError + end + + def value_color + raise NotImplementedError + end + + def key_color + '#555' + end + + def key_text_anchor + key_width / 2 + end + + def value_text_anchor + key_width + (value_width / 2) + end + + def width + key_width + value_width + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7beaecd1cf0..f4b5097adb1 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -21,7 +21,7 @@ module Gitlab private - def gl_user_id(project, bitbucket_id) + def gitlab_user_id(project, bitbucket_id) if bitbucket_id user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) (user && user.id) || project.creator_id @@ -74,7 +74,7 @@ module Gitlab description: body, title: issue["title"], state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, reporter) + author_id: gitlab_user_id(project, reporter) ) end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb new file mode 100644 index 00000000000..95308aca95f --- /dev/null +++ b/lib/gitlab/changes_list.rb @@ -0,0 +1,25 @@ +module Gitlab + class ChangesList + include Enumerable + + attr_reader :raw_changes + + def initialize(changes) + @raw_changes = changes.kind_of?(String) ? changes.lines : changes + end + + def each(&block) + changes.each(&block) + end + + def changes + @changes ||= begin + @raw_changes.map do |change| + next if change.blank? + oldrev, newrev, ref = change.strip.split(' ') + { oldrev: oldrev, newrev: newrev, ref: ref } + end.compact + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 5551fac4b8b..cb1065223d4 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -4,14 +4,14 @@ module Gitlab attr_reader :user_access, :project def initialize(change, user_access:, project:) - @oldrev, @newrev, @ref = change.split(' ') - @branch_name = branch_name(@ref) + @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) + @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project end def exec - error = protected_branch_checks || tag_checks || push_checks + error = push_checks || tag_checks || protected_branch_checks if error GitAccessStatus.new(false, error) @@ -23,6 +23,7 @@ module Gitlab protected def protected_branch_checks + return unless @branch_name return unless project.protected_branch?(@branch_name) if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) @@ -47,7 +48,7 @@ module Gitlab end def tag_checks - tag_ref = tag_name(@ref) + tag_ref = Gitlab::Git.tag_name(@ref) if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) "You are not allowed to change existing tags on this project." @@ -73,24 +74,6 @@ module Gitlab def matching_merge_request? Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? end - - def branch_name(ref) - ref = @ref.to_s - if Gitlab::Git.branch_ref?(ref) - Gitlab::Git.ref_name(ref) - else - nil - end - end - - def tag_name(ref) - ref = @ref.to_s - if Gitlab::Git.tag_ref?(ref) - Gitlab::Git.ref_name(ref) - else - nil - end - end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ae82c0db3f1..bbfa6cf7d05 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -14,7 +14,7 @@ module Gitlab @config = Loader.new(config).load! @global = Node::Global.new(@config) - @global.process! + @global.compose! end def valid? diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index 2de82d40c9d..6b7ab2fdaf2 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -23,9 +23,9 @@ module Gitlab end end - private + def compose!(deps = nil) + return unless valid? - def compose! self.class.nodes.each do |key, factory| factory .value(@config[key]) @@ -33,6 +33,12 @@ module Gitlab @entries[key] = factory.create! end + + yield if block_given? + + @entries.each_value do |entry| + entry.compose!(deps) + end end class_methods do diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 0c782c422b5..8717eabf81e 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -20,11 +20,14 @@ module Gitlab @validator.validate(:new) end - def process! + def [](key) + @entries[key] || Node::Undefined.new + end + + def compose!(deps = nil) return unless valid? - compose! - descendants.each(&:process!) + yield if block_given? end def leaf? @@ -73,11 +76,6 @@ module Gitlab def self.validator Validator end - - private - - def compose! - end end end end diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb new file mode 100644 index 00000000000..d388ab6b879 --- /dev/null +++ b/lib/gitlab/ci/config/node/environment.rb @@ -0,0 +1,68 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an environment. + # + class Environment < Entry + include Validatable + + ALLOWED_KEYS = %i[name url] + + validations do + validate do + unless hash? || string? + errors.add(:config, 'should be a hash or a string') + end + end + + validates :name, presence: true + validates :name, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message } + + validates :name, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + with_options if: :hash? do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :url, + length: { maximum: 255 }, + addressable_url: true, + allow_nil: true + end + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def url + value[:url] + end + + def value + case @config + when String then { name: @config } + when Hash then @config + else {} + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 707b052e6a8..5387f29ad59 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -37,8 +37,8 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Undefined.new( - fabricate_undefined + Node::Unspecified.new( + fabricate_unspecified ) else fabricate(@node, @value) @@ -47,13 +47,13 @@ module Gitlab private - def fabricate_undefined + def fabricate_unspecified ## # If node has a default value we fabricate concrete node # with default value. # if @node.default.nil? - fabricate(Node::Null) + fabricate(Node::Undefined) else fabricate(@node, @node.default) end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index ccd539fb003..2a2943c9288 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -36,15 +36,15 @@ module Gitlab helpers :before_script, :image, :services, :after_script, :variables, :stages, :types, :cache, :jobs - private - - def compose! - super - - compose_jobs! - compose_deprecated_entries! + def compose!(_deps = nil) + super(self) do + compose_jobs! + compose_deprecated_entries! + end end + private + def compose_jobs! factory = Node::Factory.new(Node::Jobs) .value(@config.except(*self.class.nodes.keys)) diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/node/hidden.rb index 073044b66f8..fe4ee8a7fc6 100644 --- a/lib/gitlab/ci/config/node/hidden_job.rb +++ b/lib/gitlab/ci/config/node/hidden.rb @@ -5,11 +5,10 @@ module Gitlab ## # Entry that represents a hidden CI/CD job. # - class HiddenJob < Entry + class Hidden < Entry include Validatable validations do - validates :config, type: Hash validates :config, presence: true end diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb index e84737acbb9..603334d6793 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/node/job.rb @@ -13,7 +13,7 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :environment, :dependencies + attributes :tags, :allow_failure, :when, :dependencies validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -29,58 +29,65 @@ module Gitlab inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ 'always or manual' } - validates :environment, - type: { - with: String, - message: Gitlab::Regex.environment_name_regex_message } - validates :environment, - format: { - with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } validates :dependencies, array_of_strings: true end end - node :before_script, Script, + node :before_script, Node::Script, description: 'Global before script overridden in this job.' - node :script, Commands, + node :script, Node::Commands, description: 'Commands that will be executed in this job.' - node :stage, Stage, + node :stage, Node::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Stage, + node :type, Node::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Script, + node :after_script, Node::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Cache, + node :cache, Node::Cache, description: 'Cache definition for this job.' - node :image, Image, + node :image, Node::Image, description: 'Image that will be used to execute this job.' - node :services, Services, + node :services, Node::Services, description: 'Services that will be used to execute this job.' - node :only, Trigger, + node :only, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Trigger, + node :except, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Variables, + node :variables, Node::Variables, description: 'Environment variables available for this job.' - node :artifacts, Artifacts, + node :artifacts, Node::Artifacts, description: 'Artifacts configuration for this job.' + node :environment, Node::Environment, + description: 'Environment configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts + :artifacts, :commands, :environment + + def compose!(deps = nil) + super do + if type_defined? && !stage_defined? + @entries[:stage] = @entries[:type] + end + + @entries.delete(:type) + end + + inherit!(deps) + end def name @metadata[:name] @@ -90,12 +97,30 @@ module Gitlab @config.merge(to_hash.compact) end + def commands + (before_script_value.to_a + script_value.to_a).join("\n") + end + private + def inherit!(deps) + return unless deps + + self.class.nodes.each_key do |key| + global_entry = deps[key] + job_entry = @entries[key] + + if global_entry.specified? && !job_entry.specified? + @entries[key] = global_entry + end + end + end + def to_hash { name: name, before_script: before_script, script: script, + commands: commands, image: image, services: services, stage: stage, @@ -103,19 +128,11 @@ module Gitlab only: only, except: except, variables: variables_defined? ? variables : nil, + environment: environment_defined? ? environment : nil, + environment_name: environment_defined? ? environment[:name] : nil, artifacts: artifacts, after_script: after_script } end - - def compose! - super - - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - end - - @entries.delete(:type) - end end end end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb index 51683c82ceb..d10e80d1a7d 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/node/jobs.rb @@ -26,19 +26,23 @@ module Gitlab name.to_s.start_with?('.') end - private - - def compose! - @config.each do |name, config| - node = hidden?(name) ? Node::HiddenJob : Node::Job - - factory = Node::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") + def compose!(deps = nil) + super do + @config.each do |name, config| + node = hidden?(name) ? Node::Hidden : Node::Job + + factory = Node::Factory.new(node) + .value(config || {}) + .metadata(name: name) + .with(key: name, parent: self, + description: "#{name} job definition.") + + @entries[name] = factory.create! + end - @entries[name] = factory.create! + @entries.each_value do |entry| + entry.compose!(deps) + end end end end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 88a5f53f13c..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents an undefined node. - # - # Implements the Null Object pattern. - # - class Null < Entry - def value - nil - end - - def valid? - true - end - - def errors - [] - end - - def specified? - false - end - - def relevant? - false - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb index 45fef8c3ae5..33e78023539 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -3,15 +3,34 @@ module Gitlab class Config module Node ## - # This class represents an unspecified entry node. + # This class represents an undefined node. # - # It decorates original entry adding method that indicates it is - # unspecified. + # Implements the Null Object pattern. # - class Undefined < SimpleDelegator + class Undefined < Entry + def initialize(*) + super(nil) + end + + def value + nil + end + + def valid? + true + end + + def errors + [] + end + def specified? false end + + def relevant? + false + end end end end diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/node/unspecified.rb new file mode 100644 index 00000000000..a7d1f6131b8 --- /dev/null +++ b/lib/gitlab/ci/config/node/unspecified.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an unspecified entry node. + # + # It decorates original entry adding method that indicates it is + # unspecified. + # + class Unspecified < SimpleDelegator + def specified? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb new file mode 100644 index 00000000000..a210e76acaa --- /dev/null +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -0,0 +1,141 @@ +module Gitlab + module Ci + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module PipelineDuration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest. + where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb new file mode 100644 index 00000000000..dff9e29c6a5 --- /dev/null +++ b/lib/gitlab/conflict/file.rb @@ -0,0 +1,197 @@ +module Gitlab + module Conflict + class File + include Gitlab::Routing.url_helpers + include IconsHelper + + class MissingResolution < StandardError + end + + CONTEXT_LINES = 3 + + attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository + + def initialize(merge_file_result, conflict, merge_request:) + @merge_file_result = merge_file_result + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @merge_request = merge_request + @repository = merge_request.project.repository + @match_line_headers = {} + end + + # Array of Gitlab::Diff::Line objects + def lines + @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data], + our_path: our_path, + their_path: their_path, + parent_file: self) + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line.type + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line.type == 'new' + when 'origin' + next unless line.type == 'old' + else + raise MissingResolution, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def highlight_lines! + their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + + their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + + lines.each do |line| + if line.type == 'old' + line.rich_text = their_highlight[line.old_line - 1].try(:html_safe) + else + line.rich_text = our_highlight[line.new_line - 1].try(:html_safe) + end + end + end + + def sections + return @sections if @sections + + chunked_lines = lines.chunk { |line| line.type.nil? }.to_a + match_line = nil + + sections_count = chunked_lines.size + + @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i| + section = nil + + # We need to reduce context sections to CONTEXT_LINES. Conflict sections are + # always shown in full. + if no_conflict + conflict_before = i > 0 + conflict_after = (sections_count - i) > 1 + + if conflict_before && conflict_after + # Create a gap in a long context section. + if lines.length > CONTEXT_LINES * 2 + head_lines = lines.first(CONTEXT_LINES) + tail_lines = lines.last(CONTEXT_LINES) + + # Ensure any existing match line has text for all lines up to the last + # line of its context. + update_match_line_text(match_line, head_lines.last) + + # Insert a new match line after the created gap. + match_line = create_match_line(tail_lines.first) + + section = [ + { conflict: false, lines: head_lines }, + { conflict: false, lines: tail_lines.unshift(match_line) } + ] + end + elsif conflict_after + tail_lines = lines.last(CONTEXT_LINES) + + # Create a gap and insert a match line at the start. + if lines.length > tail_lines.length + match_line = create_match_line(tail_lines.first) + + tail_lines.unshift(match_line) + end + + lines = tail_lines + elsif conflict_before + # We're at the end of the file (no conflicts after), so just remove extra + # trailing lines. + lines = lines.first(CONTEXT_LINES) + end + end + + # We want to update the match line's text every time unless we've already + # created a gap and its corresponding match line. + update_match_line_text(match_line, lines.last) unless section + + section ||= { conflict: !no_conflict, lines: lines } + section[:id] = line_code(lines.first) unless no_conflict + section + end + end + + def line_code(line) + Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + end + + def create_match_line(line) + Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) + end + + # Any line beginning with a letter, an underscore, or a dollar can be used in a + # match line header. Only context sections can contain match lines, as match lines + # have to exist in both versions of the file. + def find_match_line_header(index) + return @match_line_headers[index] if @match_line_headers.key?(index) + + @match_line_headers[index] = begin + if index >= 0 + line = lines[index] + + if line.type.nil? && line.text.match(/\A[A-Za-z$_]/) + " #{line.text}" + else + find_match_line_header(index - 1) + end + end + end + end + + # Set the match line's text for the current line. A match line takes its start + # position and context header (where present) from itself, and its end position from + # the line passed in. + def update_match_line_text(match_line, line) + return unless match_line + + header = find_match_line_header(match_line.index - 1) + + match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" + end + + def as_json(opts = nil) + { + old_path: their_path, + new_path: our_path, + blob_icon: file_type_icon_class('file', our_mode, our_path), + blob_path: namespace_project_blob_path(merge_request.project.namespace, + merge_request.project, + ::File.join(merge_request.diff_refs.head_sha, our_path)), + sections: sections + } + end + + # Don't try to print merge_request or repository. + def inspect + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode].map do |instance_variable| + value = instance_variable_get("@#{instance_variable}") + + "#{instance_variable}=\"#{value}\"" + end + + "#<#{self.class} #{instance_variables.join(' ')}>" + end + end + end +end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb new file mode 100644 index 00000000000..bbd0427a2c8 --- /dev/null +++ b/lib/gitlab/conflict/file_collection.rb @@ -0,0 +1,57 @@ +module Gitlab + module Conflict + class FileCollection + class ConflictSideMissing < StandardError + end + + attr_reader :merge_request, :our_commit, :their_commit + + def initialize(merge_request) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + end + + def repository + merge_request.project.repository + end + + def merge_index + @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + end + + def files + @files ||= merge_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), + conflict, + merge_request: merge_request) + end + end + + def as_json(opts = nil) + { + target_branch: merge_request.target_branch, + source_branch: merge_request.source_branch, + commit_sha: merge_request.diff_head_sha, + commit_message: default_commit_message, + files: files + } + end + + def default_commit_message + conflict_filenames = merge_index.conflicts.map do |conflict| + "# #{conflict[:ours][:path]}" + end + + <<EOM.chomp +Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}' + +# Conflicts: +#{conflict_filenames.join("\n")} +EOM + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb new file mode 100644 index 00000000000..98e842cded3 --- /dev/null +++ b/lib/gitlab/conflict/parser.rb @@ -0,0 +1,71 @@ +module Gitlab + module Conflict + class Parser + class ParserError < StandardError + end + + class UnexpectedDelimiter < ParserError + end + + class MissingEndDelimiter < ParserError + end + + class UnmergeableFile < ParserError + end + + class UnsupportedEncoding < ParserError + end + + def parse(text, our_path:, their_path:, parent_file: nil) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + begin + text.to_json + rescue Encoding::UndefinedConversionError + raise UnsupportedEncoding + end + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + raise UnexpectedDelimiter unless type.nil? + + type = 'new' + elsif full_line == conflict_middle + raise UnexpectedDelimiter unless type == 'new' + + type = 'old' + elsif full_line == conflict_end + raise UnexpectedDelimiter unless type == 'old' + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + else + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9dc2602867e..b164f5a2eea 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,16 +1,16 @@ module Gitlab class ContributionsCalendar - attr_reader :timestamps, :projects, :user + attr_reader :activity_dates, :projects, :user def initialize(projects, user) @projects = projects @user = user end - def timestamps - return @timestamps if @timestamps.present? + def activity_dates + return @activity_dates if @activity_dates.present? - @timestamps = {} + @activity_dates = {} date_from = 1.year.ago events = Event.reorder(nil).contributions.where(author_id: user.id). @@ -19,19 +19,17 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a + activity_dates = (1.year.ago.to_date..Date.today).to_a - dates.each do |date| - date_id = date.to_time.to_i.to_s - @timestamps[date_id] = 0 + activity_dates.each do |date| day_events = events.find { |day_events| day_events["date"] == date } if day_events - @timestamps[date_id] = day_events["total_amount"] + @activity_dates[date] = day_events["total_amount"] end end - @timestamps + @activity_dates end def events_by_date(date) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 735331df66c..ef9160d6437 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -30,6 +30,7 @@ module Gitlab signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], gravatar_enabled: Settings.gravatar['enabled'], + koding_enabled: false, sign_in_text: nil, after_sign_up_text: nil, help_page_text: nil, @@ -40,7 +41,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], + import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, @@ -58,10 +59,8 @@ module Gitlab # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false - ENV['USE_DB'] != 'false' && active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/data_builder/build.rb index 9f45aefda0f..6548e6475c6 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/data_builder/build.rb @@ -1,6 +1,8 @@ module Gitlab - class BuildDataBuilder - class << self + module DataBuilder + module Build + extend self + def build(build) project = build.project commit = build.pipeline diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/data_builder/note.rb index 8bdc89a7751..50fea1232af 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/data_builder/note.rb @@ -1,6 +1,8 @@ module Gitlab - class NoteDataBuilder - class << self + module DataBuilder + module Note + extend self + # Produce a hash of post-receive data # # For all notes: diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb new file mode 100644 index 00000000000..06a783ebc1c --- /dev/null +++ b/lib/gitlab/data_builder/pipeline.rb @@ -0,0 +1,62 @@ +module Gitlab + module DataBuilder + module Pipeline + extend self + + def build(pipeline) + { + object_kind: 'pipeline', + object_attributes: hook_attrs(pipeline), + user: pipeline.user.try(:hook_attrs), + project: pipeline.project.hook_attrs(backward: false), + commit: pipeline.commit.try(:hook_attrs), + builds: pipeline.builds.map(&method(:build_hook_attrs)) + } + end + + def hook_attrs(pipeline) + { + id: pipeline.id, + ref: pipeline.ref, + tag: pipeline.tag, + sha: pipeline.sha, + before_sha: pipeline.before_sha, + status: pipeline.status, + stages: pipeline.stages, + created_at: pipeline.created_at, + finished_at: pipeline.finished_at, + duration: pipeline.duration + } + end + + def build_hook_attrs(build) + { + id: build.id, + stage: build.stage, + name: build.name, + status: build.status, + created_at: build.created_at, + started_at: build.started_at, + finished_at: build.finished_at, + when: build.when, + manual: build.manual?, + user: build.user.try(:hook_attrs), + runner: build.runner && runner_hook_attrs(build.runner), + artifacts_file: { + filename: build.artifacts_file.filename, + size: build.artifacts_size + } + } + end + + def runner_hook_attrs(runner) + { + id: runner.id, + description: runner.description, + active: runner.active?, + is_shared: runner.is_shared? + } + end + end + end +end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/data_builder/push.rb index c8f12577112..4f81863da35 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/data_builder/push.rb @@ -1,6 +1,8 @@ module Gitlab - class PushDataBuilder - class << self + module DataBuilder + module Push + extend self + # Produce a hash of post-receive data # # data = { diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb new file mode 100644 index 00000000000..b6a89f715fd --- /dev/null +++ b/lib/gitlab/database/date_time.rb @@ -0,0 +1,27 @@ +module Gitlab + module Database + module DateTime + # Find the first of the `end_time_attrs` that isn't `NULL`. Subtract from it + # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval + # along with an alias specified by the `as` parameter. + # + # Note: For MySQL, the interval is returned in seconds. + # For PostgreSQL, the interval is returned as an INTERVAL type. + def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) + diff_fn = if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + + query_so_far.project(diff_fn.as(as)) + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb new file mode 100644 index 00000000000..1444d25ebc7 --- /dev/null +++ b/lib/gitlab/database/median.rb @@ -0,0 +1,112 @@ +# https://www.periscopedata.com/blog/medians-in-sql.html +module Gitlab + module Database + module Median + def median_datetime(arel_table, query_so_far, column_sym) + median_queries = + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym) + elsif Gitlab::Database.mysql? + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + + results = Array.wrap(median_queries).map do |query| + ActiveRecord::Base.connection.execute(query) + end + extract_median(results).presence + end + + def extract_median(results) + result = results.compact.first + + if Gitlab::Database.postgresql? + result = result.first.presence + median = result['median'] if result + median.to_f if median + elsif Gitlab::Database.mysql? + result.to_a.flatten.first + end + end + + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + query = arel_table. + from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). + project(average([arel_table[column_sym]], 'median')). + where( + Arel::Nodes::Between.new( + Arel.sql("(select @row_id := @row_id + 1)"), + Arel::Nodes::And.new( + [Arel.sql('@ct/2.0'), + Arel.sql('@ct/2.0 + 1')] + ) + ) + ). + # Disallow negative values + where(arel_table[column_sym].gteq(0)) + + [ + Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), + Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), + Arel.sql("set @row_id := 0;"), + query.to_sql, + Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") + ] + end + + def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + # Create a CTE with the column we're operating on, row number (after sorting by the column + # we're operating on), and count of the table we're operating on (duplicated across) all rows + # of the CTE. For example, if we're looking to find the median of the `projects.star_count` + # column, the CTE might look like this: + # + # star_count | row_id | ct + # ------------+--------+---- + # 5 | 1 | 3 + # 9 | 2 | 3 + # 15 | 3 | 3 + cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( + cte_table, + arel_table. + project( + arel_table[column_sym].as(column_sym.to_s), + Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), + Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), + arel_table.project("COUNT(1)").as('ct')). + # Disallow negative values + where(arel_table[column_sym].gteq(zero_interval))) + + # From the CTE, select either the middle row or the middle two rows (this is accomplished + # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the + # selected rows, and this is the median value. + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). + where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) + ) + ). + with(query_so_far, cte). + to_sql + end + + private + + def average(args, as) + Arel::Nodes::NamedFunction.new("AVG", args, as) + end + + def extract_epoch(arel_attribute) + Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) + end + + # Need to cast '0' to an INTERVAL before we can check if the interval is positive + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 927f9dad20b..0bd6e148ba8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -129,12 +129,14 @@ module Gitlab # column - The name of the column to add. # type - The column type (e.g. `:integer`). # default - The default value for the column. + # limit - Sets a column limit. For example, for :integer, the default is + # 4-bytes. Set `limit: 8` to allow 8-byte integers. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. # # This method can also take a block which is passed directly to the # `update_column_in_batches` method. - def add_column_with_default(table, column, type, default:, allow_null: false, &block) + def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -144,7 +146,11 @@ module Gitlab disable_statement_timeout transaction do - add_column(table, column, type, default: nil) + if limit + add_column(table, column, type, default: nil, limit: limit) + else + add_column(table, column, type, default: nil) + end # Changing the default before the update ensures any newly inserted # rows already use the proper default value. diff --git a/lib/gitlab/diff/file_collection/merge_request.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 4f946908e2f..36348b33943 100644 --- a/lib/gitlab/diff/file_collection/merge_request.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -1,14 +1,14 @@ module Gitlab module Diff module FileCollection - class MergeRequest < Base - def initialize(merge_request, diff_options:) - @merge_request = merge_request + class MergeRequestDiff < Base + def initialize(merge_request_diff, diff_options:) + @merge_request_diff = merge_request_diff - super(merge_request, - project: merge_request.project, + super(merge_request_diff, + project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request.diff_refs) + diff_refs: merge_request_diff.diff_refs) end def diff_files @@ -61,11 +61,11 @@ module Gitlab end def cacheable? - @merge_request.merge_request_diff.present? + @merge_request_diff.present? end def cache_key - [@merge_request.merge_request_diff, 'highlighted-diff-files', diff_options] + [@merge_request_diff, 'highlighted-diff-files', diff_options] end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index cf097e0d0de..80a146b4a5a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,11 +2,13 @@ module Gitlab module Diff class Line attr_reader :type, :index, :old_pos, :new_pos + attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos + @parent_file = parent_file end def self.init_from_hash(hash) @@ -43,9 +45,25 @@ module Gitlab type == 'old' end + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text + end + def meta? type == 'match' || type == 'nonewline' end + + def as_json(opts = nil) + { + type: type, + old_line: old_line, + new_line: new_line, + text: text, + rich_text: rich_text || text + } + end end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 2fdcf8d7838..ecf62dead35 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -139,13 +139,19 @@ module Gitlab private def find_diff_file(repository) - diffs = Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ).diffs(paths: paths) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + compare = Gitlab::Git::Commit.find(repository.raw_repository, head_sha) + else + compare = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ) + end + + diff = compare.diffs(paths: paths).first - diff = diffs.first return unless diff Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 4446e921e0d..40a4815a9a0 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -1,10 +1,10 @@ module Gitlab class DowntimeCheck class Message - attr_reader :path, :offline, :reason + attr_reader :path, :offline - OFFLINE = "\e[32moffline\e[0m" - ONLINE = "\e[31monline\e[0m" + OFFLINE = "\e[31moffline\e[0m" + ONLINE = "\e[32monline\e[0m" # path - The file path of the migration. # offline - When set to `true` the migration will require downtime. @@ -19,10 +19,21 @@ module Gitlab label = offline ? OFFLINE : ONLINE message = "[#{label}]: #{path}" - message += ": #{reason}" if reason + + if reason? + message += ":\n\n#{reason}\n\n" + end message end + + def reason? + @reason.present? + end + + def reason + @reason.strip.lines.map(&:strip).join("\n") + end end end end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index bd3267e2a80..5cf9d5ebe28 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -4,7 +4,8 @@ require 'gitlab/email/handler/create_issue_handler' module Gitlab module Email module Handler - HANDLERS = [CreateNoteHandler, CreateIssueHandler] + # The `CreateIssueHandler` feature is disabled for the time being. + HANDLERS = [CreateNoteHandler] def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index b7ed11cb638..7cccf465334 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -45,6 +45,7 @@ module Gitlab def verify_record!(record:, invalid_exception:, record_name:) return if record.persisted? + return if record.errors.key?(:commands_only) error_title = "The #{record_name} could not be created for the following reasons:" diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 191bea86ac3..3cd515e4a3a 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,34 @@ module Gitlab ref.gsub(/\Arefs\/(tags|heads)\//, '') end + def branch_name(ref) + ref = ref.to_s + if self.branch_ref?(ref) + self.ref_name(ref) + else + nil + end + end + + def committer_hash(email:, name:) + return if email.nil? || name.nil? + + { + email: email, + name: name, + time: Time.now + } + end + + def tag_name(ref) + ref = ref.to_s + if self.tag_ref?(ref) + self.ref_name(ref) + else + nil + end + end + def tag_ref?(ref) ref.start_with?(TAG_REF_PREFIX) end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 9b681e636c7..bd90d24a2ec 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -17,11 +17,13 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return [true, nil] unless exists? - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + Bundler.with_clean_env do + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 69943e22353..799794c0171 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -5,12 +5,13 @@ module Gitlab DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project, :protocol, :user_access + attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol + @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) end @@ -60,14 +61,26 @@ module Gitlab end def user_download_access_check - unless user_access.can_do_action?(:download_code) + unless user_can_download_code? || build_can_download_code? return build_status_object(false, "You are not allowed to download code from this project.") end build_status_object(true) end + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + def user_push_access_check(changes) + unless authentication_abilities.include?(:push_code) + return build_status_object(false, "You are not allowed to upload code for this project.") + end + if changes.blank? return build_status_object(true) end @@ -76,10 +89,10 @@ module Gitlab return build_status_object(false, "A repository for this project does not exist yet.") end - changes = changes.lines if changes.kind_of?(String) + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied - changes.map(&:strip).reject(&:blank?).each do |change| + changes_list.each do |change| status = change_access_check(change) unless status.allowed? # If user does not have access to make at least one change - cancel all push @@ -134,7 +147,7 @@ module Gitlab end def build_status_object(status, message = '') - GitAccessStatus.new(status, message) + Gitlab::GitAccessStatus.new(status, message) end end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 72992baffd4..8cacf4f4925 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,11 +15,16 @@ module Gitlab private - def gl_user_id(github_id) + def gitlab_user_id(github_id) User.joins(:identities). find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). try(:id) end + + def gitlab_author_id + return @gitlab_author_id if defined?(@gitlab_author_id) + @gitlab_author_id = gitlab_user_id(raw_data.user.id) + end end end end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 7d2d545b84e..4750675ae9d 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -7,10 +7,6 @@ module Gitlab branch_exists? && commit_exists? end - def name - @name ||= exists? ? ref : "#{ref}-#{short_id}" - end - def valid? repo.present? end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 084e514492c..e33ac61f5ae 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -52,7 +52,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - request { api.send(method, *args, &block) } + request(method, *args, &block) else super(method, *args, &block) end @@ -99,20 +99,19 @@ module Gitlab rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME end - def request + def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = yield + data = api.send(method, *args, &block) + yield data last_response = api.last_response while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get - data.concat(last_response.data) if last_response.data.is_a?(Array) + yield last_response.data if last_response.data.is_a?(Array) end - - data end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 2c1b94ef2cd..2bddcde2b7c 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -21,7 +21,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -52,7 +52,11 @@ module Gitlab end def note - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def type diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb deleted file mode 100644 index db1fabaa18a..00000000000 --- a/lib/gitlab/github_import/hook_formatter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module GithubImport - class HookFormatter - EVENTS = %w[* create delete pull_request push].freeze - - attr_reader :raw - - delegate :id, :name, :active, to: :raw - - def initialize(raw) - @raw = raw - end - - def config - raw.config.attrs - end - - def valid? - (EVENTS & raw.events).any? && active - end - end - end -end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 3932fcb1eda..b8321244473 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,24 +3,33 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :project, :repo, :repo_url + attr_reader :client, :errors, :project, :repo, :repo_url def initialize(project) @project = project @repo = project.import_source @repo_url = project.import_url + @errors = [] + @labels = {} if credentials @client = Client.new(credentials[:user]) - @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" end end def execute - import_labels && import_milestones && import_issues && - import_pull_requests && import_wiki + import_labels + import_milestones + import_issues + import_pull_requests + import_comments + import_wiki + import_releases + handle_errors + + true end private @@ -29,140 +38,134 @@ module Gitlab @credentials ||= project.import_data.credentials if project.import_data end - def import_labels - labels = client.labels(repo, per_page: 100) - labels.each { |raw| LabelFormatter.new(project, raw).create! } + def handle_errors + return unless errors.any? - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) end - def import_milestones - milestones = client.milestones(repo, state: :all, per_page: 100) - milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } + def import_labels + client.labels(repo, per_page: 100) do |labels| + labels.each do |raw| + begin + label = LabelFormatter.new(project, raw).create! + @labels[label.title] = label.id + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end + end - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + def import_milestones + client.milestones(repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + MilestoneFormatter.new(project, raw).create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end end def import_issues - issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - - issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) - - if gh_issue.valid? - issue = gh_issue.create! - apply_labels(issue) - import_comments(issue) if gh_issue.has_comments? + client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw) + + if gh_issue.valid? + begin + issue = gh_issue.create! + apply_labels(issue, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end end - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message end def import_pull_requests - disable_webhooks - - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) - - source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } - target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } - branches_removed = source_branches_removed | target_branches_removed - - restore_branches(branches_removed) - - pull_requests.each do |pull_request| - merge_request = pull_request.create! - apply_labels(merge_request) - import_comments(merge_request) - import_comments_on_diff(merge_request) + client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + pull_requests.each do |raw| + pull_request = PullRequestFormatter.new(project, raw) + next unless pull_request.valid? + + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request = pull_request.create! + apply_labels(merge_request, raw) + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(pull_request) + end + end end - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message - ensure - clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks - end - - def disable_webhooks - update_webhooks(hooks, active: false) - end - - def clean_up_disabled_webhooks - update_webhooks(hooks, active: true) + project.repository.after_remove_branch end - def update_webhooks(hooks, options) - hooks.each do |hook| - client.edit_hook(repo, hook.id, hook.name, hook.config, options) - end + def restore_source_branch(pull_request) + project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name) end - def hooks - @hooks ||= - begin - client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - - # The GitHub Repository Webhooks API returns 404 for users - # without admin access to the repository when listing hooks. - # In this case we just want to return gracefully instead of - # spitting out an error and stop the import process. - rescue Octokit::NotFound - [] - end + def restore_target_branch(pull_request) + project.repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) end - def restore_branches(branches) - branches.each do |name, sha| - client.create_ref(repo, "refs/heads/#{name}", sha) - end - - project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') + def remove_branch(name) + project.repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :remove_branch, name: name } end - def clean_up_restored_branches(branches) - branches.each do |name, _| - client.delete_ref(repo, "heads/#{name}") - project.repository.delete_branch(name) rescue Rugged::ReferenceError - end - - project.repository.after_remove_branch + def clean_up_restored_branches(pull_request) + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end - def apply_labels(issuable) - issue = client.issue(repo, issuable.iid) - - if issue.labels.count > 0 - label_ids = issue.labels.map do |raw| - Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) - end + def apply_labels(issuable, raw_issuable) + if raw_issuable.labels.count > 0 + label_ids = raw_issuable.labels + .map { |attrs| @labels[attrs.name] } + .compact issuable.update_attribute(:label_ids, label_ids) end end - def import_comments(issuable) - comments = client.issue_comments(repo, issuable.iid, per_page: 100) - create_comments(issuable, comments) - end + def import_comments + client.issues_comments(repo, per_page: 100) do |comments| + create_comments(comments, :issue) + end - def import_comments_on_diff(merge_request) - comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) - create_comments(merge_request, comments) + client.pull_requests_comments(repo, per_page: 100) do |comments| + create_comments(comments, :pull_request) + end end - def create_comments(issuable, comments) - comments.each do |raw| - comment = CommentFormatter.new(project, raw) - issuable.notes.create!(comment.attributes) + def create_comments(comments, issuable_type) + ActiveRecord::Base.no_touching do + comments.each do |raw| + begin + comment = CommentFormatter.new(project, raw) + issuable_class = issuable_type == :issue ? Issue : MergeRequest + iid = raw.send("#{issuable_type}_url").split('/').last # GH doesn't return parent ID directly + issuable = issuable_class.find_by_iid(iid) + next unless issuable + + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end end @@ -170,18 +173,27 @@ module Gitlab unless project.wiki_enabled? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.update_attribute(:wiki_enabled, true) + project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED) end - - true rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - raise Projects::ImportService::Error, e.message - else - true + errors << { type: :wiki, errors: e.message } + end + end + + def import_releases + client.releases(repo, per_page: 100) do |releases| + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 835ec858b35..77621de9f4c 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -12,7 +12,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -40,7 +40,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -49,7 +49,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -57,7 +57,11 @@ module Gitlab end def description - @formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -69,10 +73,6 @@ module Gitlab def state raw_data.state == 'closed' ? 'closed' : 'opened' end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 9f18244e7d7..2cad7fca88e 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -13,6 +13,12 @@ module Gitlab Label end + def create! + project.labels.find_or_create_by!(title: title) do |label| + label.color = color + end + end + private def color diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 53d4b3102d1..b2fa524cf5b 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,14 +3,14 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: number, + iid: raw_data.number, project: project, - title: title, - description: description, - due_date: due_date, + title: raw_data.title, + description: raw_data.description, + due_date: raw_data.due_on, state: state, - created_at: created_at, - updated_at: updated_at + created_at: raw_data.created_at, + updated_at: raw_data.updated_at } end @@ -20,33 +20,9 @@ module Gitlab private - def number - raw_data.number - end - - def title - raw_data.title - end - - def description - raw_data.description - end - - def due_date - raw_data.due_on - end - def state raw_data.state == 'closed' ? 'closed' : 'active' end - - def created_at - raw_data.created_at - end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index f4221003db5..605abfabdab 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -3,26 +3,33 @@ module Gitlab class ProjectCreator attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data end def execute - ::Projects::CreateService.new( + project = ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.name, + name: @name, + path: @name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), - wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") ).execute + + # If repo has wiki we'll import it later + if repo.has_wiki? && project + project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) + end + + project end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index a4ea2210abd..1408683100f 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,8 +1,8 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter - delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true + delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true def attributes { @@ -20,7 +20,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -33,17 +33,33 @@ module Gitlab end def valid? - source_branch.valid? && target_branch.valid? && !cross_project? + source_branch.valid? && target_branch.valid? end def source_branch @source_branch ||= BranchFormatter.new(project, raw_data.head) end + def source_branch_name + @source_branch_name ||= begin + source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}" + end + end + def target_branch @target_branch ||= BranchFormatter.new(project, raw_data.base) end + def target_branch_name + @target_branch_name ||= begin + target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}" + end + end + + def url + raw_data.url + end + private def assigned? @@ -52,7 +68,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -61,19 +77,19 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body raw_data.body || "" end - def cross_project? - source_branch_repo.id != target_branch_repo.id - end - def description - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -91,15 +107,6 @@ module Gitlab 'opened' end end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at - else - raw_data.updated_at - end - end end end end diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb new file mode 100644 index 00000000000..73d643b00ad --- /dev/null +++ b/lib/gitlab/github_import/release_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class ReleaseFormatter < BaseFormatter + def attributes + { + project: project, + tag: raw_data.tag_name, + description: raw_data.body, + created_at: raw_data.created_at, + updated_at: raw_data.created_at + } + end + + def klass + Release + end + + def valid? + !raw_data.draft + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 46d40f75be6..e44d7934fda 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -41,7 +41,8 @@ module Gitlab title: issue["title"], state: issue["state"], updated_at: issue["updated_at"], - author_id: gl_user_id(project, issue["author"]["id"]) + author_id: gitlab_user_id(project, issue["author"]["id"]), + confidential: issue["confidential"] ) end end @@ -51,7 +52,7 @@ module Gitlab private - def gl_user_id(project, gitlab_id) + def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end diff --git a/lib/gitlab/gitorious_import.rb b/lib/gitlab/gitorious_import.rb deleted file mode 100644 index 8d0132a744c..00000000000 --- a/lib/gitlab/gitorious_import.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module GitoriousImport - GITORIOUS_HOST = "https://gitorious.org" - end -end diff --git a/lib/gitlab/gitorious_import/client.rb b/lib/gitlab/gitorious_import/client.rb deleted file mode 100644 index 99fe5bdebfc..00000000000 --- a/lib/gitlab/gitorious_import/client.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module GitoriousImport - class Client - attr_reader :repo_list - - def initialize(repo_list) - @repo_list = repo_list - end - - def authorize_url(redirect_uri) - "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}" - end - - def repos - @repos ||= repo_names.map { |full_name| GitoriousImport::Repository.new(full_name) } - end - - def repo(id) - repos.find { |repo| repo.id == id } - end - - private - - def repo_names - repo_list.to_s.split(',').map(&:strip).reject(&:blank?) - end - end - end -end diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb deleted file mode 100644 index 8e22aa9286d..00000000000 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module GitoriousImport - class ProjectCreator - attr_reader :repo, :namespace, :current_user - - def initialize(repo, namespace, current_user) - @repo = repo - @namespace = namespace - @current_user = current_user - end - - def execute - ::Projects::CreateService.new( - current_user, - name: repo.name, - path: repo.path, - description: repo.description, - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel::PUBLIC, - import_type: "gitorious", - import_source: repo.full_name, - import_url: repo.import_url - ).execute - end - end - end -end diff --git a/lib/gitlab/gitorious_import/repository.rb b/lib/gitlab/gitorious_import/repository.rb deleted file mode 100644 index c88f1ae358d..00000000000 --- a/lib/gitlab/gitorious_import/repository.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module GitoriousImport - Repository = Struct.new(:full_name) do - def id - Digest::SHA1.hexdigest(full_name) - end - - def namespace - segments.first - end - - def path - segments.last - end - - def name - path.titleize - end - - def description - "" - end - - def import_url - "#{GITORIOUS_HOST}/#{full_name}.git" - end - - private - - def segments - full_name.split('/') - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c5a11148d33..2c21804fe7a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,7 +11,6 @@ module Gitlab if current_user gon.current_user_id = current_user.id - gon.api_token = current_user.private_token end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb562bdcd2c..181e288a014 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.3' + # For every version update, the version history in import_export.md has to be kept up to date. + VERSION = '0.1.4' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1da51043611..bb9d1080330 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,5 +1,8 @@ # Model relationships to be included in the project import/export project_tree: + - :labels + - milestones: + - :events - issues: - :events - notes: @@ -10,6 +13,7 @@ project_tree: - milestone: - :events - snippets: + - :award_emoji - notes: :author - :releases @@ -35,19 +39,15 @@ project_tree: - :deploy_keys - :services - :hooks - - :protected_branches - - :labels - - milestones: - - :events + - protected_branches: + - :merge_access_levels + - :push_access_levels + - :project_feature # Only include the following attributes for the models specified. included_attributes: project: - :description - - :issues_enabled - - :merge_requests_enabled - - :wiki_enabled - - :snippets_enabled - :visibility_level - :archived user: @@ -67,9 +67,13 @@ excluded_attributes: - :milestone_id merge_requests: - :milestone_id + award_emoji: + - :awardable_id methods: statuses: - :type + services: + - :type merge_request_diff: - - :utf8_st_diffs
\ No newline at end of file + - :utf8_st_diffs diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 008300bde45..0cc10f40087 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -57,19 +57,16 @@ module Gitlab # +value+ existing model to be included in the hash # +json_config_hash+ the original hash containing the root model def create_model_value(current_key, value, json_config_hash) - parsed_hash = { include: value } - parse_hash(value, parsed_hash) - - json_config_hash[current_key] = parsed_hash + json_config_hash[current_key] = parse_hash(value) || { include: value } end # Calls attributes finder to parse the hash and add any attributes to it # # +value+ existing model to be included in the hash # +parsed_hash+ the original hash - def parse_hash(value, parsed_hash) + def parse_hash(value) @attributes_finder.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } + { include: hash_or_merge(value, hash) } end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7b3551b84c..35ff134ea19 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -61,11 +61,17 @@ module Gitlab def restore_project return @project unless @tree_hash - project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } @project.update(project_params) @project end + def project_params + @tree_hash.reject do |key, value| + # return params that are not 1 to many or 1 to 1 relations + value.is_a?(Array) || key == key.singularize + end + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b0726268ca6..354ccd64696 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -7,7 +7,9 @@ module Gitlab variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze @@ -17,6 +19,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + FINDER_ATTRIBUTES = %w[title project_id].freeze + def self.create(*args) new(*args).create end @@ -149,7 +153,7 @@ module Gitlab end def parsed_relation_hash - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } end def set_st_diffs @@ -161,14 +165,30 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) - existing_object.assign_attributes(parsed_relation_hash) + events = parsed_relation_hash.delete('events') + + unless events.blank? + existing_object.assign_attributes(events: events) + end + existing_object else relation_class.new(parsed_relation_hash) end end end + + def existing_object + @existing_object ||= + begin + finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES) + existing_object = relation_class.find_or_create_by(finder_hash) + # Done in two steps, as MySQL behaves differently than PostgreSQL using + # the +find_or_create_by+ method and does not return the ID the second time. + existing_object.update(parsed_relation_hash) + existing_object + end + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 6d9379acf25..d1e33ea8678 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -22,10 +22,6 @@ module Gitlab private - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def path_to_repo @project.repository.path_to_repo end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index de3fe6d822e..fc08082fc86 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,8 +24,8 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 59a05411fe9..94261b7eeed 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -14,13 +14,12 @@ module Gitlab def options { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Gitorious.org' => 'gitorious', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', + 'GitHub' => 'github', + 'Bitbucket' => 'bitbucket', + 'GitLab.com' => 'gitlab', + 'Google Code' => 'google_code', + 'FogBugz' => 'fogbugz', + 'Repo by URL' => 'git', 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 2f326d00a2f..7e06bd2b0fb 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -51,8 +51,6 @@ module Gitlab user.ldap_block false end - rescue - false end def adapter diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 9a5bcfb5c9b..8b38cfaefb6 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -23,31 +23,7 @@ module Gitlab end def users(field, value, limit = nil) - if field.to_sym == :dn - options = { - base: value, - scope: Net::LDAP::SearchScope_BaseObject - } - else - options = { - base: config.base, - filter: Net::LDAP::Filter.eq(field, value) - } - end - - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - - options[:filter] = if options[:filter] - Net::LDAP::Filter.join(options[:filter], user_filter) - else - user_filter - end - end - - if limit.present? - options.merge!(size: limit) - end + options = user_options(field, value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -86,10 +62,49 @@ module Gitlab results end end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] rescue Timeout::Error Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") [] end + + private + + def user_options(field, value, limit) + options = { attributes: user_attributes } + options[:size] = limit if limit + + if field.to_sym == :dn + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + options[:filter] = user_filter + else + options[:base] = config.base + options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + end + + options + end + + def user_filter(filter = nil) + if config.user_filter.present? + user_filter = Net::LDAP::Filter.construct(config.user_filter) + end + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end + + def user_attributes + %W(#{config.uid} cn mail dn) + end end end end diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb deleted file mode 100644 index a1ee1aa81ff..00000000000 --- a/lib/gitlab/lfs/response.rb +++ /dev/null @@ -1,329 +0,0 @@ -module Gitlab - module Lfs - class Response - def initialize(project, user, ci, request) - @origin_project = project - @project = storage_project(project) - @user = user - @ci = ci - @env = request.env - @request = request - end - - def render_download_object_response(oid) - render_response_to_download do - if check_download_sendfile_header? - render_lfs_sendfile(oid) - else - render_not_found - end - end - end - - def render_batch_operation_response - request_body = JSON.parse(@request.body.read) - case request_body["operation"] - when "download" - render_batch_download(request_body) - when "upload" - render_batch_upload(request_body) - else - render_not_found - end - end - - def render_storage_upload_authorize_response(oid, size) - render_response_to_push do - [ - 200, - { "Content-Type" => "application/json; charset=utf-8" }, - [JSON.dump({ - 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", - 'LfsOid' => oid, - 'LfsSize' => size - })] - ] - end - end - - def render_storage_upload_store_response(oid, size, tmp_file_name) - return render_forbidden unless tmp_file_name - - render_response_to_push do - render_lfs_upload_ok(oid, size, tmp_file_name) - end - end - - def render_unsupported_deprecated_api - [ - 501, - { "Content-Type" => "application/json; charset=utf-8" }, - [JSON.dump({ - 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - private - - def render_not_enabled - [ - 501, - { - "Content-Type" => "application/json; charset=utf-8", - }, - [JSON.dump({ - 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_unauthorized - [ - 401, - { - 'Content-Type' => 'text/plain' - }, - ['Unauthorized'] - ] - end - - def render_not_found - [ - 404, - { - "Content-Type" => "application/vnd.git-lfs+json" - }, - [JSON.dump({ - 'message' => 'Not found.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_forbidden - [ - 403, - { - "Content-Type" => "application/vnd.git-lfs+json" - }, - [JSON.dump({ - 'message' => 'Access forbidden. Check your access level.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_lfs_sendfile(oid) - return render_not_found unless oid.present? - - lfs_object = object_for_download(oid) - - if lfs_object && lfs_object.file.exists? - [ - 200, - { - # GitLab-workhorse will forward Content-Type header - "Content-Type" => "application/octet-stream", - "X-Sendfile" => lfs_object.file.path - }, - [] - ] - else - render_not_found - end - end - - def render_batch_upload(body) - return render_not_found if body.empty? || body['objects'].nil? - - render_response_to_push do - response = build_upload_batch_response(body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] - end - end - - def render_batch_download(body) - return render_not_found if body.empty? || body['objects'].nil? - - render_response_to_download do - response = build_download_batch_response(body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] - end - end - - def render_lfs_upload_ok(oid, size, tmp_file) - if store_file(oid, size, tmp_file) - [ - 200, - { - 'Content-Type' => 'text/plain', - 'Content-Length' => 0 - }, - [] - ] - else - [ - 422, - { 'Content-Type' => 'text/plain' }, - ["Unprocessable entity"] - ] - end - end - - def render_response_to_download - return render_not_enabled unless Gitlab.config.lfs.enabled - - unless @project.public? - return render_unauthorized unless @user || @ci - return render_forbidden unless user_can_fetch? - end - - yield - end - - def render_response_to_push - return render_not_enabled unless Gitlab.config.lfs.enabled - return render_unauthorized unless @user - return render_forbidden unless user_can_push? - - yield - end - - def check_download_sendfile_header? - @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" - end - - def user_can_fetch? - # Check user access against the project they used to initiate the pull - @ci || @user.can?(:download_code, @origin_project) - end - - def user_can_push? - # Check user access against the project they used to initiate the push - @user.can?(:push_code, @origin_project) - end - - def storage_project(project) - if project.forked? - storage_project(project.forked_from_project) - else - project - end - end - - def store_file(oid, size, tmp_file) - tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) - - object = LfsObject.find_or_create_by(oid: oid, size: size) - if object.file.exists? - success = true - else - success = move_tmp_file_to_storage(object, tmp_file_path) - end - - if success - success = link_to_project(object) - end - - success - ensure - # Ensure that the tmp file is removed - FileUtils.rm_f(tmp_file_path) - end - - def object_for_download(oid) - @project.lfs_objects.find_by(oid: oid) - end - - def move_tmp_file_to_storage(object, path) - File.open(path) do |f| - object.file = f - end - - object.file.store! - object.save - end - - def link_to_project(object) - if object && !object.projects.exists?(@project.id) - object.projects << @project - object.save - end - end - - def select_existing_objects(objects) - objects_oids = objects.map { |o| o['oid'] } - @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set - end - - def build_upload_batch_response(objects) - selected_objects = select_existing_objects(objects) - - upload_hypermedia_links(objects, selected_objects) - end - - def build_download_batch_response(objects) - selected_objects = select_existing_objects(objects) - - download_hypermedia_links(objects, selected_objects) - end - - def download_hypermedia_links(all_objects, existing_objects) - all_objects.each do |object| - if existing_objects.include?(object['oid']) - object['actions'] = { - 'download' => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - else - object['error'] = { - 'code' => 404, - 'message' => "Object does not exist on the server or you don't have permissions to access it", - } - end - end - - { 'objects' => all_objects } - end - - def upload_hypermedia_links(all_objects, existing_objects) - all_objects.each do |object| - # generate actions only for non-existing objects - next if existing_objects.include?(object['oid']) - - object['actions'] = { - 'upload' => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - end - - { 'objects' => all_objects } - end - end - end -end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb deleted file mode 100644 index f2a76a56b8f..00000000000 --- a/lib/gitlab/lfs/router.rb +++ /dev/null @@ -1,98 +0,0 @@ -module Gitlab - module Lfs - class Router - attr_reader :project, :user, :ci, :request - - def initialize(project, user, ci, request) - @project = project - @user = user - @ci = ci - @env = request.env - @request = request - end - - def try_call - return unless @request && @request.path.present? - - case @request.request_method - when 'GET' - get_response - when 'POST' - post_response - when 'PUT' - put_response - else - nil - end - end - - private - - def get_response - path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) - return nil unless path_match - - oid = path_match[2] - return nil unless oid - - case path_match[1] - when "info/lfs" - lfs.render_unsupported_deprecated_api - when "gitlab-lfs" - lfs.render_download_object_response(oid) - else - nil - end - end - - def post_response - post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) - return nil unless post_path - - # Check for Batch API - if post_path[0].ends_with?("/info/lfs/objects/batch") - lfs.render_batch_operation_response - elsif post_path[0].ends_with?("/info/lfs/objects") - lfs.render_unsupported_deprecated_api - else - nil - end - end - - def put_response - object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) - return nil if object_match.nil? - - oid = object_match[1] - size = object_match[2].try(:to_i) - return nil if oid.nil? || size.nil? - - # GitLab-workhorse requests - # 1. Try to authorize the request - # 2. send a request with a header containing the name of the temporary file - if object_match[3] && object_match[3] == '/authorize' - lfs.render_storage_upload_authorize_response(oid, size) - else - tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) - lfs.render_storage_upload_store_response(oid, size, tmp_file_name) - end - end - - def lfs - return unless @project - - Gitlab::Lfs::Response.new(@project, @user, @ci, @request) - end - - def sanitize_tmp_filename(name) - if name.present? - name.gsub!(/^.*(\\|\/)/, '') - name = name.match(/[0-9a-f]{73}/) - name[0] if name - else - nil - end - end - end - end -end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb new file mode 100644 index 00000000000..5f67e97fa2a --- /dev/null +++ b/lib/gitlab/lfs_token.rb @@ -0,0 +1,48 @@ +module Gitlab + class LfsToken + attr_accessor :actor + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 1800 + + def initialize(actor) + @actor = + case actor + when DeployKey, User + actor + when Key + actor.user + else + raise 'Bad Actor' + end + end + + def token + Gitlab::Redis.with do |redis| + token = redis.get(redis_key) + token ||= Devise.friendly_token(TOKEN_LENGTH) + redis.set(redis_key, token, ex: EXPIRY_TIME) + + token + end + end + + def user? + actor.is_a?(User) + end + + def type + actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + end + + def actor_name + actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + end + + private + + def redis_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb new file mode 100644 index 00000000000..12999a90a29 --- /dev/null +++ b/lib/gitlab/mail_room.rb @@ -0,0 +1,47 @@ +require 'yaml' +require 'json' +require_relative 'redis' unless defined?(Gitlab::Redis) + +module Gitlab + module MailRoom + class << self + def enabled? + config[:enabled] && config[:address] + end + + def config + @config ||= fetch_config + end + + def reset_config! + @config = nil + end + + private + + def fetch_config + return {} unless File.exist?(config_file) + + rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys + + config = all_config[:incoming_email] || {} + config[:enabled] = false if config[:enabled].nil? + config[:port] = 143 if config[:port].nil? + config[:ssl] = false if config[:ssl].nil? + config[:start_tls] = false if config[:start_tls].nil? + config[:mailbox] = 'inbox' if config[:mailbox].nil? + + if config[:enabled] && config[:address] + config[:redis_url] = Gitlab::Redis.new(rails_env).url + end + + config + end + + def config_file + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + end + end + end +end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 41fcd971c22..3d1ba33ec68 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -124,6 +124,15 @@ module Gitlab trans.action = action if trans end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + trans = current_transaction + + trans.add_event(*args) if trans + end + # Returns the prefix to use for the name of a series. def self.series_prefix @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f23d67e1e38..bd0afe53c51 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,15 +4,20 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags + attr_reader :series, :values, :tags, :type # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}) + def initialize(series, values, tags = {}, type = :metric) @values = values @series = series @tags = tags + @type = type + end + + def event? + type == :event end # Returns a Hash in a format that can be directly written to InfluxDB. diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e61670f491c..01c96a6fe96 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,6 +4,17 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' + CONTENT_TYPES = { + 'text/html' => :html, + 'text/plain' => :txt, + 'application/json' => :json, + 'text/js' => :js, + 'application/atom+xml' => :atom, + 'image/png' => :png, + 'image/jpeg' => :jpeg, + 'image/gif' => :gif, + 'image/svg+xml' => :svg + } def initialize(app) @app = app @@ -17,6 +28,10 @@ module Gitlab begin retval = trans.run { @app.call(env) } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:rails_exception) + + raise error # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure @@ -42,8 +57,15 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - trans.action = "#{controller.class.name}##{controller.action_name}" + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + suffix = CONTENT_TYPES[controller.content_type] + + if suffix && suffix != :html + action += ".#{suffix}" + end + + trans.action = action end def tag_endpoint(trans, env) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index a1240fd33ee..f9dd8e41912 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,10 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:sidekiq_exception) + + raise error ensure trans.finish end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 968f3218950..7bc16181be6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,10 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values, :methods + # The series to store events (e.g. Git pushes) in. + EVENT_SERIES = 'events' + + attr_reader :tags, :values, :method, :metrics attr_accessor :action @@ -55,6 +58,20 @@ module Gitlab @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) end + # Tracks a business level event + # + # Business level events including events such as Git pushes, Emails being + # sent, etc. + # + # event_name - The name of the event (e.g. "git_push"). + # tags - A set of tags to attach to the event. + def add_event(event_name, tags = {}) + @metrics << Metric.new(EVENT_SERIES, + { count: 1 }, + { event: event_name }.merge(tags), + :event) + end + # Returns a MethodCall object for the given name. def method_call_for(name) unless method = @methods[name] @@ -101,7 +118,7 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - hash[:tags][:action] ||= @action if @action + hash[:tags][:action] ||= @action if @action && !metric.event? hash end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 56608b1b276..5d2d7d0026c 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -11,7 +11,7 @@ module Gitlab def call(env) trans = Gitlab::Metrics.current_transaction - proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index ca23ccef25b..cc74bb29087 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -18,18 +18,18 @@ module Gitlab FileUtils.mkdir_p(path) end - @cmd_output = "" - @cmd_status = 0 + cmd_output = "" + cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - # We are not using stdin so we should close it, in case the command we - # are running waits for input. + yield(stdin) if block_given? stdin.close - @cmd_output << stdout.read - @cmd_output << stderr.read - @cmd_status = wait_thr.value.exitstatus + + cmd_output << stdout.read + cmd_output << stderr.read + cmd_status = wait_thr.value.exitstatus end - [@cmd_output, @cmd_status] + [cmd_output, cmd_status] end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 183bd10d6a3..5b9cfaeb2f8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -28,11 +28,6 @@ module Gitlab end end - def total_count - @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + commits_count - end - def blobs_count @blobs_count ||= blobs.count end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 1f92986ec9a..3faab937726 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -1,50 +1,89 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' + module Gitlab class Redis CACHE_NAMESPACE = 'cache:gitlab' SESSION_NAMESPACE = 'session:gitlab' SIDEKIQ_NAMESPACE = 'resque:gitlab' + MAILROOM_NAMESPACE = 'mail_room:gitlab' + DEFAULT_REDIS_URL = 'redis://localhost:6379' + CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) - attr_reader :url + class << self + # Do NOT cache in an instance variable. Result may be mutated by caller. + def params + new.params + end - # To be thread-safe we must be careful when writing the class instance - # variables @url and @pool. Because @pool depends on @url we need two - # mutexes to prevent deadlock. - URL_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :URL_MUTEX, :POOL_MUTEX + # Do NOT cache in an instance variable. Result may be mutated by caller. + # @deprecated Use .params instead to get sentinel support + def url + new.url + end - def self.url - @url || URL_MUTEX.synchronize { @url = new.url } - end + def with + @pool ||= ConnectionPool.new { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end - def self.with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(url: url) } + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = File.read(CONFIG_FILE).freeze + rescue Errno::ENOENT + @_raw_config = false end + + @_raw_config end - @pool.with { |redis| yield redis } end - def self.redis_store_options - url = new.url - redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) end - redis_config_hash end - def initialize(rails_env = nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) + def raw_config_hash + config_data = fetch_config - @url = "redis://localhost:6379" - if File.exist?(config_file) - @url = YAML.load_file(config_file)[rails_env] + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: DEFAULT_REDIS_URL } end end + + def fetch_config + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false + end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ffad5e17c78..776bbcbb5d0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -44,7 +44,7 @@ module Gitlab end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'." end def file_path_regex @@ -52,7 +52,7 @@ module Gitlab end def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." end def directory_traversal_regex @@ -60,7 +60,7 @@ module Gitlab end def directory_traversal_regex_message - "cannot include directory traversal. " + "cannot include directory traversal." end def archive_formats_regex @@ -96,11 +96,11 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index f8ab2b1f09e..2690938fe82 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -27,11 +27,6 @@ module Gitlab end end - def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + - milestones_count - end - def projects_count @projects_count ||= projects.count end @@ -48,10 +43,6 @@ module Gitlab @milestones_count ||= milestones.count end - def empty? - total_count.zero? - end - private def projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb new file mode 100644 index 00000000000..117fc508135 --- /dev/null +++ b/lib/gitlab/sentry.rb @@ -0,0 +1,27 @@ +module Gitlab + module Sentry + def self.enabled? + Rails.env.production? && current_application_settings.sentry_enabled? + end + + def self.context(current_user = nil) + return unless self.enabled? + + if current_user + Raven.user_context( + id: current_user.id, + email: current_user.email, + username: current_user.username, + ) + end + end + + def self.program_context + if Sidekiq.server? + 'sidekiq' + else + 'rails' + end + end + end +end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb new file mode 100644 index 00000000000..60d35be2599 --- /dev/null +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -0,0 +1,57 @@ +module Gitlab + module SlashCommands + class CommandDefinition + attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + + def initialize(name, attributes = {}) + @name = name + + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @params = attributes[:params] || [] + @condition_block = attributes[:condition_block] + @action_block = attributes[:action_block] + end + + def all_names + [name, *aliases] + end + + def noop? + action_block.nil? + end + + def available?(opts) + return true unless condition_block + + context = OpenStruct.new(opts) + context.instance_exec(&condition_block) + end + + def execute(context, opts, arg) + return if noop? || !available?(opts) + + if arg.present? + context.instance_exec(arg, &action_block) + elsif action_block.arity == 0 + context.instance_exec(&action_block) + end + end + + def to_h(opts) + desc = description + if desc.respond_to?(:call) + context = OpenStruct.new(opts) + desc = context.instance_exec(&desc) rescue '' + end + + { + name: name, + aliases: aliases, + description: desc, + params: params + } + end + end + end +end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb new file mode 100644 index 00000000000..50b0937d267 --- /dev/null +++ b/lib/gitlab/slash_commands/dsl.rb @@ -0,0 +1,98 @@ +module Gitlab + module SlashCommands + module Dsl + extend ActiveSupport::Concern + + included do + cattr_accessor :command_definitions, instance_accessor: false do + [] + end + + cattr_accessor :command_definitions_by_name, instance_accessor: false do + {} + end + end + + class_methods do + # Allows to give a description to the next slash command. + # This description is shown in the autocomplete menu. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # desc do + # "This is a dynamic description for #{noteable.to_ability_name}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def desc(text = '', &block) + @description = block_given? ? block : text + end + + # Allows to define params for the next slash command. + # These params are shown in the autocomplete menu. + # + # Example: + # + # params "~label ~label2" + # command :command_key do |arguments| + # # Awesome code block + # end + def params(*params) + @params = params + end + + # Allows to define conditions that must be met in order for the command + # to be returned by `.command_names` & `.command_definitions`. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # condition do + # project.public? + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def condition(&block) + @condition_block = block + end + + # Registers a new command which is recognizeable from body of email or + # comment. + # It accepts aliases and takes a block. + # + # Example: + # + # command :my_command, :alias_for_my_command do |arguments| + # # Awesome code block + # end + def command(*command_names, &block) + name, *aliases = command_names + + definition = CommandDefinition.new( + name, + aliases: aliases, + description: @description, + params: @params, + condition_block: @condition_block, + action_block: block + ) + + self.command_definitions << definition + + definition.all_names.each do |name| + self.command_definitions_by_name[name] = definition + end + + @description = nil + @params = nil + @condition_block = nil + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb new file mode 100644 index 00000000000..a672e5e4855 --- /dev/null +++ b/lib/gitlab/slash_commands/extractor.rb @@ -0,0 +1,122 @@ +module Gitlab + module SlashCommands + # This class takes an array of commands that should be extracted from a + # given text. + # + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # ``` + class Extractor + attr_reader :command_definitions + + def initialize(command_definitions) + @command_definitions = command_definitions + end + + # Extracts commands from content and return an array of commands. + # The array looks like the following: + # [ + # ['command1'], + # ['command3', 'arg1 arg2'], + # ] + # The command and the arguments are stripped. + # The original command text is removed from the given `content`. + # + # Usage: + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) + # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] + # msg #=> "hello\nworld" + # ``` + def extract_commands(content, opts = {}) + return [content, []] unless content + + content = content.dup + + commands = [] + + content.delete!("\r") + content.gsub!(commands_regex(opts)) do + if $~[:cmd] + commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + '' + else + $~[0] + end + end + + [content.strip, commands] + end + + private + + # Builds a regular expression to match known commands. + # First match group captures the command name and + # second match group captures its arguments. + # + # It looks something like: + # + # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ + def commands_regex(opts) + names = command_names(opts).map(&:to_s) + + @commands_regex ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `/cmd arg` which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `/cmd arg` which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?<html> + # Quote block: + # >>> + # Anything, including `/cmd arg` which are ignored by this filter + # >>> + + ^>>> + .+? + \n>>>$ + ) + | + (?: + # Command not in a blockquote, blockcode, or HTML tag: + # /close + + ^\/ + (?<cmd>#{Regexp.union(names)}) + (?: + [ ] + (?<arg>[^\/\n]*) + )? + (?:\n|$) + ) + }mx + end + + def command_names(opts) + command_definitions.flat_map do |command| + next if command.noop? + + command.all_names + end.compact + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e0e74ff8359..9e01f02029c 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -20,10 +20,6 @@ module Gitlab end end - def total_count - @total_count ||= snippet_titles_count + snippet_blobs_count - end - def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 760ff3e614a..7ebec8e2cff 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,8 +1,9 @@ module Gitlab module Template class BaseTemplate - def initialize(path) + def initialize(path, project = nil) @path = path + @finder = self.class.finder(project) end def name @@ -10,23 +11,32 @@ module Gitlab end def content - File.read(@path) + @finder.read(@path) + end + + def to_json + { name: name, content: content } end class << self - def all - self.categories.keys.flat_map { |cat| by_category(cat) } + def all(project = nil) + if categories.any? + categories.keys.flat_map { |cat| by_category(cat, project) } + else + by_category("", project) + end end - def find(key) - file_name = "#{key}#{self.extension}" - - directory = select_directory(file_name) - directory ? new(File.join(category_directory(directory), file_name)) : nil + def find(key, project = nil) + path = self.finder(project).find(key) + path.present? ? new(path, project) : nil end + # Set categories as sub directories + # Example: { "category_name_1" => "directory_path_1", "category_name_2" => "directory_name_2" } + # Default is no category with all files in base dir of each class def categories - raise NotImplementedError + {} end def extension @@ -37,29 +47,40 @@ module Gitlab raise NotImplementedError end - def by_category(category) - templates_for_directory(category_directory(category)) + # Defines which strategy will be used to get templates files + # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject + # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects + def finder(project = nil) + raise NotImplementedError end - def category_directory(category) - File.join(base_dir, categories[category]) + def by_category(category, project = nil) + directory = category_directory(category) + files = finder(project).list_files_for(directory) + + files.map { |f| new(f, project) } end - private + def category_directory(category) + return base_dir unless category.present? - def select_directory(file_name) - categories.keys.find do |category| - File.exist?(File.join(category_directory(category), file_name)) - end + File.join(base_dir, categories[category]) end - def templates_for_directory(dir) - dir << '/' unless dir.end_with?('/') - Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) } - end + # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] } + # If no category is present returns [{ name: template_name }, { name: template2_name}] + def dropdown_names(project = nil) + return [] if project && !project.repository.exists? - def filter_regex - @filter_reges ||= /#{Regexp.escape(extension)}\z/ + if categories.any? + categories.keys.map do |category| + files = self.by_category(category, project) + [category, files.map { |t| { name: t.name } }] + end.to_h + else + files = self.all(project) + files.map { |t| { name: t.name } } + end end end end diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb new file mode 100644 index 00000000000..473b05257c6 --- /dev/null +++ b/lib/gitlab/template/finders/base_template_finder.rb @@ -0,0 +1,35 @@ +module Gitlab + module Template + module Finders + class BaseTemplateFinder + def initialize(base_dir) + @base_dir = base_dir + end + + def list_files_for + raise NotImplementedError + end + + def read + raise NotImplementedError + end + + def find + raise NotImplementedError + end + + def category_directory(category) + return @base_dir unless category.present? + + @base_dir + @categories[category] + end + + class << self + def filter_regex(extension) + /#{Regexp.escape(extension)}\z/ + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb new file mode 100644 index 00000000000..831da45191f --- /dev/null +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -0,0 +1,38 @@ +# Searches and reads file present on Gitlab installation directory +module Gitlab + module Template + module Finders + class GlobalTemplateFinder < BaseTemplateFinder + def initialize(base_dir, extension, categories = {}) + @categories = categories + @extension = extension + super(base_dir) + end + + def read(path) + File.read(path) + end + + def find(key) + file_name = "#{key}#{@extension}" + + directory = select_directory(file_name) + directory ? File.join(category_directory(directory), file_name) : nil + end + + def list_files_for(dir) + dir << '/' unless dir.end_with?('/') + Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + @categories.keys.find do |category| + File.exist?(File.join(category_directory(category), file_name)) + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb new file mode 100644 index 00000000000..22c39436cb2 --- /dev/null +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -0,0 +1,59 @@ +# Searches and reads files present on each Gitlab project repository +module Gitlab + module Template + module Finders + class RepoTemplateFinder < BaseTemplateFinder + # Raised when file is not found + class FileNotFoundError < StandardError; end + + def initialize(project, base_dir, extension, categories = {}) + @categories = categories + @extension = extension + @repository = project.repository + @commit = @repository.head_commit if @repository.exists? + + super(base_dir) + end + + def read(path) + blob = @repository.blob_at(@commit.id, path) if @commit + raise FileNotFoundError if blob.nil? + blob.data + end + + def find(key) + file_name = "#{key}#{@extension}" + directory = select_directory(file_name) + raise FileNotFoundError if directory.nil? + + category_directory(directory) + file_name + end + + def list_files_for(dir) + return [] unless @commit + + dir << '/' unless dir.end_with?('/') + + entries = @repository.tree(:head, dir).entries + + names = entries.map(&:name) + names.select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + return [] unless @commit + + # Insert root as directory + directories = ["", @categories.keys] + + directories.find do |category| + path = category_directory(category) + file_name + @repository.blob_at(@commit.id, path) + end + end + end + end + end +end diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore_template.rb index 964fbfd4de3..8d2a9d2305c 100644 --- a/lib/gitlab/template/gitignore.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class Gitignore < BaseTemplate + class GitignoreTemplate < BaseTemplate class << self def extension '.gitignore' @@ -16,6 +16,10 @@ module Gitlab def base_dir Rails.root.join('vendor/gitignore') end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 7f480fe33c0..8d1a1ed54c9 100644 --- a/lib/gitlab/template/gitlab_ci_yml.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class GitlabCiYml < BaseTemplate + class GitlabCiYmlTemplate < BaseTemplate def content explanation = "# This file is a template, and might need editing before it works on your project." [explanation, super].join("\n") @@ -21,6 +21,10 @@ module Gitlab def base_dir Rails.root.join('vendor/gitlab-ci-yml') end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end end end end diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb new file mode 100644 index 00000000000..c6fa8d3eafc --- /dev/null +++ b/lib/gitlab/template/issue_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class IssueTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/issue_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb new file mode 100644 index 00000000000..f826c02f3b5 --- /dev/null +++ b/lib/gitlab/template/merge_request_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class MergeRequestTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/merge_request_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index fe65c246101..99d0c28e749 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -22,6 +22,8 @@ module Gitlab note_url when WikiPage wiki_page_url + when ProjectSnippet + project_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index c55a7fc4d3d..9858d2e7d83 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -32,7 +32,7 @@ module Gitlab if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - access_levels = project.protected_branches.matching(ref).map(&:push_access_level) + access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten access_levels.any? { |access_level| access_level.check_access(user) } else user.can?(:push_code, project) @@ -43,7 +43,7 @@ module Gitlab return false unless user if project.protected_branch?(ref) - access_levels = project.protected_branches.matching(ref).map(&:merge_access_level) + access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten access_levels.any? { |access_level| access_level.check_access(user) } else user.can?(:push_code, project) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index d13fe0ef8a9..e59ead5d76c 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -7,7 +7,7 @@ module Gitlab # @param cmd [Array<String>] # @return [Boolean] def system_silent(cmd) - Popen::popen(cmd).last.zero? + Popen.popen(cmd).last.zero? end def force_utf8(str) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c6826a09bd2..5d33f98e89e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,19 +1,38 @@ require 'base64' require 'json' +require 'securerandom' module Gitlab class Workhorse SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' + + # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 + # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + SECRET_LENGTH = 32 class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'RepoPath' => repository.path_to_repo, + GL_ID: Gitlab::GlId.gl_id(user), + RepoPath: repository.path_to_repo, + } + end + + def lfs_upload_ok(oid, size) + { + StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + LfsOid: oid, + LfsSize: size, } end + def artifact_upload_ok + { TempPath: ArtifactUploader.artifacts_upload_path } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -41,7 +60,7 @@ module Gitlab def send_git_diff(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -54,7 +73,7 @@ module Gitlab def send_git_patch(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -81,6 +100,35 @@ module Gitlab path.readable? ? path.read.chomp : 'unknown' end + def secret + @secret ||= begin + bytes = Base64.strict_decode64(File.read(secret_path).chomp) + raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes + end + end + + def write_secret + bytes = SecureRandom.random_bytes(SECRET_LENGTH) + File.open(secret_path, 'w:BINARY', 0600) do |f| + f.chmod(0600) + f.write(Base64.strict_encode64(bytes)) + end + end + + def verify_api_request!(request_headers) + JWT.decode( + request_headers[INTERNAL_API_REQUEST_HEADER], + secret, + true, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + ) + end + + def secret_path + Rails.root.join('.gitlab_workhorse_secret') + end + protected def encode(hash) diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 60f4636e737..5f4a6bbfa35 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -46,7 +46,7 @@ namespace :gitlab do } correct_options = options.map do |name, value| - run(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value end if correct_options.all? @@ -64,7 +64,7 @@ namespace :gitlab do for_more_information( see_installation_guide_section "GitLab" ) - end + end end end @@ -73,7 +73,7 @@ namespace :gitlab do database_config_file = Rails.root.join("config", "database.yml") - if File.exists?(database_config_file) + if File.exist?(database_config_file) puts "yes".color(:green) else puts "no".color(:red) @@ -94,7 +94,7 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") - if File.exists?(gitlab_config_file) + if File.exist?(gitlab_config_file) puts "yes".color(:green) else puts "no".color(:red) @@ -113,7 +113,7 @@ namespace :gitlab do print "GitLab config outdated? ... " gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exists?(gitlab_config_file) + unless File.exist?(gitlab_config_file) puts "can't check because of previous errors".color(:magenta) end @@ -144,7 +144,7 @@ namespace :gitlab do script_path = "/etc/init.d/gitlab" - if File.exists?(script_path) + if File.exist?(script_path) puts "yes".color(:green) else puts "no".color(:red) @@ -169,7 +169,7 @@ namespace :gitlab do recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") script_path = "/etc/init.d/gitlab" - unless File.exists?(script_path) + unless File.exist?(script_path) puts "can't check because of previous errors".color(:magenta) return end @@ -316,7 +316,7 @@ namespace :gitlab do min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " - redis_version = run(%W(redis-cli --version)) + redis_version = run_command(%W(redis-cli --version)) redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) @@ -361,7 +361,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - if File.exists?(repo_base_path) + if File.exist?(repo_base_path) puts "yes".color(:green) else puts "no".color(:red) @@ -385,7 +385,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -408,7 +408,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -438,7 +438,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -893,7 +893,7 @@ namespace :gitlab do def check_ruby_version required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run(%W(ruby --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version))) print "Ruby version >= #{required_version} ? ... " @@ -910,7 +910,7 @@ namespace :gitlab do def check_git_version required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run(%W(#{Gitlab.config.git.bin_path} --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" print "Git version >= #{required_version} ? ... " diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index fe43d40e6d2..dffea8ed155 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -8,7 +8,7 @@ namespace :gitlab do # check Ruby version ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s) # check Gem version - gem_version = run(%W(gem --version)) + gem_version = run_command(%W(gem --version)) # check Bundler version bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) # check Bundler version @@ -17,7 +17,7 @@ namespace :gitlab do puts "" puts "System information".color(:yellow) puts "System:\t\t#{os_name || "unknown".color(:red)}" - puts "Current User:\t#{run(%W(whoami))}" + puts "Current User:\t#{run_command(%W(whoami))}" puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ba93945bd03..bb7eb852f1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -90,7 +90,7 @@ namespace :gitlab do task build_missing_projects: :environment do Project.find_each(batch_size: 1000) do |project| path_to_repo = project.repository.path_to_repo - if File.exists?(path_to_repo) + if File.exist?(path_to_repo) print '-' else if Gitlab::Shell.new.add_repository(project.repository_storage_path, diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ab96b1d3593..74be413423a 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -23,7 +23,7 @@ namespace :gitlab do # It will primarily use lsb_relase to determine the OS. # It has fallbacks to Debian, SuSE, OS X and systems running systemd. def os_name - os_name = run(%W(lsb_release -irs)) + os_name = run_command(%W(lsb_release -irs)) os_name ||= if File.readable?('/etc/system-release') File.read('/etc/system-release') end @@ -34,7 +34,7 @@ namespace :gitlab do os_name ||= if File.readable?('/etc/SuSE-release') File.read('/etc/SuSE-release') end - os_name ||= if os_x_version = run(%W(sw_vers -productVersion)) + os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) "Mac OS X #{os_x_version}" end os_name ||= if File.readable?('/etc/os-release') @@ -62,10 +62,10 @@ namespace :gitlab do # Returns nil if nothing matched # Returns the MatchData if the pattern matched # - # see also #run + # see also #run_command # see also String#match def run_and_match(command, regexp) - run(command).try(:match, regexp) + run_command(command).try(:match, regexp) end # Runs the given command @@ -74,7 +74,7 @@ namespace :gitlab do # Returns the output of the command otherwise # # see also #run_and_match - def run(command) + def run_command(command) output, _ = Gitlab::Popen.popen(command) output rescue Errno::ENOENT @@ -82,7 +82,7 @@ namespace :gitlab do end def uid_for(user_name) - run(%W(id -u #{user_name})).chomp.to_i + run_command(%W(id -u #{user_name})).chomp.to_i end def gid_for(group_name) @@ -96,7 +96,7 @@ namespace :gitlab do def warn_user_is_not_gitlab unless @warned_user_not_gitlab gitlab_user = Gitlab.config.gitlab.user - current_user = run(%W(whoami)).chomp + current_user = run_command(%W(whoami)).chomp unless current_user == gitlab_user puts " Warning ".color(:black).background(:yellow) puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake new file mode 100644 index 00000000000..609dfaa48e3 --- /dev/null +++ b/lib/tasks/haml-lint.rake @@ -0,0 +1,5 @@ +unless Rails.env.production? + require 'haml_lint/rake_task' + + HamlLint::RakeTask.new +end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index da255f5464b..8dbfa7751dc 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -34,21 +34,19 @@ task :spinach do run_spinach_tests(nil) end -def run_command(cmd) +def run_system_command(cmd) system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) end def run_spinach_command(args) - run_command(%w(spinach -r rerun) + args) + run_system_command(%w(spinach -r rerun) + args) end def run_spinach_tests(tags) - #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!') - success = run_spinach_command(%W(--tags #{tags})) 3.times do |_| break if success - break unless File.exists?('tmp/spinach-rerun.txt') + break unless File.exist?('tmp/spinach-rerun.txt') tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) puts '' |