diff options
Diffstat (limited to 'lib')
102 files changed, 1851 insertions, 1043 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 9d1d9058996..87915b19480 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -5,32 +5,27 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project access requests viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # GET /groups/:id/access_requests - # GET /projects/:id/access_requests + desc "Gets a list of access requests for a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end get ":id/access_requests" do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - access_requesters = paginate(source.requesters.includes(:user)) + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end - # Request access to the group/project - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # POST /groups/:id/access_requests - # POST /projects/:id/access_requests + desc "Requests access for the authenticated user to a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end post ":id/access_requests" do source = find_source(source_type, params[:id]) access_requester = source.request_access(current_user) @@ -42,42 +37,34 @@ module API end end - # Approve a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # access_level (optional) - Access level - # - # Example Request: - # PUT /groups/:id/access_requests/:user_id/approve - # PUT /projects/:id/access_requests/:user_id/approve + desc 'Approves an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + end put ':id/access_requests/:user_id/approve' do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute status :created present member.user, with: Entities::Member, member: member end - # Deny a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # - # Example Request: - # DELETE /groups/:id/access_requests/:user_id - # DELETE /projects/:id/access_requests/:user_id + desc 'Denies an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + end delete ":id/access_requests/:user_id" do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - access_requester = source.requesters.find_by!(user_id: params[:user_id]) - - ::Members::DestroyService.new(access_requester, current_user).execute + ::Members::DestroyService.new(source, current_user, params). + execute(:requesters) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index cb47ec8f33f..67109ceeef9 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -31,11 +31,12 @@ module API # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji + mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages mount ::API::Builds - mount ::API::CommitStatuses mount ::API::Commits + mount ::API::CommitStatuses mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -45,19 +46,18 @@ module API mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::LicenseTemplates mount ::API::Lint mount ::API::Members - mount ::API::MergeRequests mount ::API::MergeRequestDiffs + mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectSnippets mount ::API::Projects + mount ::API::ProjectSnippets mount ::API::Repositories mount ::API::Runners mount ::API::Services @@ -72,5 +72,10 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::Version + + route :any, '*path' do + error!('404 Not Found', 404) + end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2461a783ea8..e9ccba3b465 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -8,16 +8,19 @@ module API awardable_string = awardable_type.pluralize awardable_id_string = "#{awardable_type}_id" + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) @@ -27,14 +30,13 @@ module API end end - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end get "#{endpoint}/:award_id" do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji @@ -43,17 +45,14 @@ module API end end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/award_emoji + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end post endpoint do - required_attributes! [:name] - not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -65,14 +64,13 @@ module API end end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) diff --git a/lib/api/boards.rb b/lib/api/boards.rb new file mode 100644 index 00000000000..4ac491edc1b --- /dev/null +++ b/lib/api/boards.rb @@ -0,0 +1,132 @@ +module API + # Boards API + class Boards < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + get ':id/boards' do + authorize!(:read_board, user_project) + present user_project.boards, with: Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `backlog` and `done` lists. This feature was introduced in 8.13' + success Entities::List + end + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: Entities::List + end + + desc 'Get a list of a project board' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + end + get '/lists/:list_id' do + authorize!(:read_board, user_project) + present board_lists.find(params[:list_id]), with: Entities::List + end + + desc 'Create a new board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end + post '/lists' do + unless available_labels.exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::CreateService.new(user_project, current_user, + { label_id: params[:label_id] }) + + list = service.execute(project_board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + desc 'Moves a board list to a new position' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + requires :position, type: Integer, desc: 'The position of the list' + end + put '/lists/:list_id' do + list = project_board.lists.movable.find(params[:list_id]) + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::MoveService.new(user_project, current_user, + { position: params[:position] }) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_project) + + list = board_lists.find(params[:list_id]) + + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b615703df93..6d827448994 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -54,43 +54,25 @@ module API not_found!('Branch') unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) - protected_branch_params = { - name: @branch.name + name: @branch.name, + developers_can_push: to_boolean(params[:developers_can_push]), + developers_can_merge: to_boolean(params[:developers_can_merge]) } - # If `developers_can_merge` is switched off, _all_ `DEVELOPER` - # merge_access_levels need to be deleted. - if developers_can_merge == false - protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + service_args = [user_project, current_user, protected_branch_params] - # If `developers_can_push` is switched off, _all_ `DEVELOPER` - # push_access_levels need to be deleted. - if developers_can_push == false - protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - protected_branch_params.merge!( - merge_access_levels_attributes: [{ - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }], - push_access_levels_attributes: [{ - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }] - ) - - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present @branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end # Unprotect a single branch @@ -123,7 +105,7 @@ module API post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], @@ -142,10 +124,10 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 52bdbcae5a8..67adca6605f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -3,15 +3,32 @@ module API class Builds < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ['pending', 'running', 'failed', 'success', 'canceled'], + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +37,13 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +57,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +72,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +86,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +103,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +123,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +140,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +158,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +175,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -235,14 +239,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index dfbdd597d29..f54d4f06627 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -6,17 +6,17 @@ module API resource :projects do before { authenticate! } - # Get a commit's statuses - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # stage (optional) - The stage - # name (optional) - The name - # all (optional) - Show all statuses, default: false - # Examples: - # GET /projects/:id/repository/commits/:sha/statuses + desc "Get a commit's statuses" do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + optional :ref, type: String, desc: 'The ref' + optional :stage, type: String, desc: 'The stage' + optional :name, type: String, desc: 'The name' + optional :all, type: String, desc: 'Show all statuses, default: false' + end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -31,22 +31,23 @@ module API present paginate(statuses), with: Entities::CommitStatus end - # Post status to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, failed or canceled - # target_url (optional) - The target URL to associate with this status - # description (optional) - A short description of the status - # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" - # Examples: - # POST /projects/:id/statuses/:sha + desc 'Post status to a commit' do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + requires :state, type: String, desc: 'The state of the status', + values: ['pending', 'running', 'success', 'failed', 'canceled'] + optional :ref, type: String, desc: 'The ref' + optional :target_url, type: String, desc: 'The target URL to associate with this status' + optional :description, type: String, desc: 'A short description of the status' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + end post ':id/statuses/:sha' do authorize! :create_commit_status, user_project - required_attributes! [:state] - attrs = attributes_for_keys [:target_url, :description] + commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -66,9 +67,14 @@ module API pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) status = GenericCommitStatus.running_or_pending.find_or_initialize_by( - project: @project, pipeline: pipeline, - user: current_user, name: name, ref: ref) - status.attributes = attrs + project: @project, + pipeline: pipeline, + user: current_user, + name: name, + ref: ref, + target_url: params[:target_url], + description: params[:description] + ) begin case params[:state].to_s diff --git a/lib/api/commits.rb b/lib/api/commits.rb index b4eaf1813d4..2f2cf769481 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -6,102 +6,149 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository commits - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # since (optional) - Only commits after or in this date will be returned - # until (optional) - Only commits before or in this date will be returned - # Example Request: - # GET /projects/:id/repository/commits + desc 'Get a project repository commits' do + success Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: String, desc: 'Only commits after or in this date will be returned' + optional :until, type: String, desc: 'Only commits before or in this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end get ":id/repository/commits" do + # TODO remove the next line for 9.0, use DateTime type in the params block datetime_attributes! :since, :until - page = (params[:page] || 0).to_i - per_page = (params[:per_page] || 20).to_i ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - after = params[:since] - before = params[:until] + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) - commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before) present commits, with: Entities::RepoCommit end - # Get a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash or name of a repository branch or tag - # Example Request: - # GET /projects/:id/repository/commits/:sha + desc 'Commit multiple file changes as one commit' do + success Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :id, type: Integer, desc: 'The project ID' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array, desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared(params) + attrs[:source_branch] = attrs[:branch_name] + attrs[:target_branch] = attrs[:branch_name] + attrs[:actions].map! do |action| + action[:action] = action[:action].to_sym + action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/') + action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/') + action + end + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + present commit, with: Entities::RepoCommitDetail end - # Get the diff for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # Example Request: - # GET /projects/:id/repository/commits/:sha/diff + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha/diff" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + commit.raw_diffs.to_a end - # Get a commit's comments - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # Examples: - # GET /projects/:id/repository/commits/:sha/comments + desc "Get a commit's comments" do + success Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :per_page, type: Integer, desc: 'The amount of items per page for paginaion' + optional :page, type: Integer, desc: 'The page number for pagination' + end get ':id/repository/commits/:sha/comments' do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit notes = Note.where(commit_id: commit.id).order(:created_at) + present paginate(notes), with: Entities::CommitNote end - # Post comment to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # note (required) - Text of comment - # path (optional) - The file path - # line (optional) - The line number - # line_type (optional) - The type of line (new or old) - # Examples: - # POST /projects/:id/repository/commits/:sha/comments + desc 'Post comment to commit' do + success Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: ['new', 'old'], default: 'new', desc: 'The type of the line' + end + end post ':id/repository/commits/:sha/comments' do - required_attributes! [:note] - - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit + opts = { note: params[:note], noteable_type: 'Commit', commit_id: commit.id } - if params[:path] && params[:line] && params[:line_type] + if params[:path] commit.raw_diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| - next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + next unless line.new_pos == params[:line] && line.type == params[:line_type] break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b081cb8e520..e1995aa6f4b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -344,7 +344,7 @@ module API end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -433,8 +433,11 @@ module API end end - class Label < Grape::Entity + class LabelBasic < Grape::Entity expose :name, :color, :description + end + + class Label < LabelBasic expose :open_issues_count, :closed_issues_count, :open_merge_requests_count expose :subscribed do |label, options| @@ -442,6 +445,19 @@ module API end end + class List < Grape::Entity + expose :id + expose :label, using: Entities::LabelBasic + expose :position + end + + class Board < Grape::Entity + expose :id + expose :lists, using: Entities::List do |board| + board.lists.destroyable + end + end + class Compare < Grape::Entity expose :commit, using: Entities::RepoCommit do |compare, options| Commit.decorate(compare.commits, nil).last @@ -495,6 +511,8 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :koding_enabled + expose :koding_url end class Release < Grape::Entity @@ -546,6 +564,10 @@ module API expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -553,6 +575,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -563,8 +586,8 @@ module API expose :key, :value end - class Pipeline < Grape::Entity - expose :id, :status, :ref, :sha, :before_sha, :tag, :yaml_errors + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e9abd53e62b..8ca4cfa68cd 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -6,6 +6,8 @@ module API resource :groups do # Get a groups list # + # Parameters: + # skip_groups (optional) - Array of group ids to exclude from list # Example Request: # GET /groups get do @@ -16,6 +18,7 @@ module API end @groups = @groups.search(params[:search]) if params[:search].present? + @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? @groups = paginate @groups present @groups, with: Entities::Group end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9d71e90075a..d05841def89 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -21,8 +21,11 @@ module API end # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints def find_user_from_warden - warden ? warden.authenticate : nil + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end def find_user_by_private_token @@ -68,6 +71,10 @@ module API @project ||= find_project(params[:id]) end + def available_labels + @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + end + def find_project(id) project = Project.find_with_namespace(id) || Project.find_by(id: id) @@ -115,7 +122,7 @@ module API end def find_project_label(id) - label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') end @@ -194,16 +201,11 @@ module API def validate_label_params(params) errors = {} - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) + params[:labels].to_s.split(',').each do |label_name| + label = available_labels.find_or_initialize_by(title: label_name.strip) + next if label.valid? - if label.invalid? - errors[label.title] = label.errors - end - end + errors[label.title] = label.errors end errors @@ -430,7 +432,7 @@ module API end def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp + Gitlab::Shell.secret_token end def send_git_blob(repository, blob) diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2b723b79504..767f27ef334 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -4,10 +4,9 @@ module API before { authenticate! } resource :keys do - # Get single ssh key by id. Only available to admin users. - # - # Example Request: - # GET /keys/:id + desc 'Get single ssh key by id. Only available to admin users' do + success Entities::SSHKeyWithUser + end get ":id" do authenticated_as_admin! diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c806829d69e..326e1e7ae00 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,37 +3,32 @@ module API class Labels < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end get ':id/labels' do - present user_project.labels, with: Entities::Label, current_user: current_user + present available_labels, with: Entities::Label, current_user: current_user end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = user_project.find_label(attrs[:name]) + label = user_project.find_label(params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + label = user_project.labels.create(declared(params, include_parent_namespaces: false).to_h) if label.valid? present label, with: Entities::Label, current_user: current_user @@ -42,54 +37,44 @@ module API end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + update_params['name'] = update_params.delete('new_name') if update_params.key?('new_name') - if label.update(attrs) + if label.update(update_params) present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) diff --git a/lib/api/license_templates.rb b/lib/api/license_templates.rb deleted file mode 100644 index d0552299ed0..00000000000 --- a/lib/api/license_templates.rb +++ /dev/null @@ -1,58 +0,0 @@ -module API - # License Templates API - class LicenseTemplates < Grape::API - PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze - - # Get the list of the available license templates - # - # Parameters: - # popular - Filter licenses to only the popular ones - # - # Example Request: - # GET /licenses - # GET /licenses?popular=1 - get 'licenses' do - options = { - featured: params[:popular].present? ? true : nil - } - present Licensee::License.all(options), with: Entities::RepoLicense - end - - # Get text for specific license - # - # Parameters: - # key (required) - The key of a license - # project - Copyrighted project name - # fullname - Full name of copyright holder - # - # Example Request: - # GET /licenses/mit - # - get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do - required_attributes! [:key] - - not_found!('License') unless Licensee::License.find(params[:key]) - - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - license = Licensee::License.new(params[:key]) - - license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - - present license, with: Entities::RepoLicense - end - end -end diff --git a/lib/api/members.rb b/lib/api/members.rb index 37f0a6512f4..b80818f0eb6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -5,16 +5,16 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project members viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # query - Query string - # - # Example Request: - # GET /groups/:id/members - # GET /projects/:id/members + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + end get ":id/members" do source = find_source(source_type, params[:id]) @@ -25,15 +25,12 @@ module API present users, with: Entities::Member, source: source end - # Get a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # GET /groups/:id/members/:user_id - # GET /projects/:id/members/:user_id + desc 'Gets a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -43,48 +40,34 @@ module API present member.user, with: Entities::Member, member: member end - # Add a new group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the new member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # POST /groups/:id/members - # POST /projects/:id/members + desc 'Adds a member to a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] - - access_requester = source.requesters.find_by(user_id: params[:user_id]) - if access_requester - # We pass current_user = access_requester so that the requester doesn't - # receive a "access denied" email - ::Members::DestroyService.new(access_requester, access_requester.user).execute - end member = source.members.find_by(user_id: params[:user_id]) - # This is to ensure back-compatibility but 409 behavior should be used - # for both project and group members in 9.0! + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! conflict!('Member already exists') if source_type == 'group' && member unless member - source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - member = source.members.find_by(user_id: params[:user_id]) + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end - if member + if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # Since `source.add_user` doesn't return a member object, we have to - # build a new one and populate its errors in order to render them. - member = source.members.build(attributes_for_keys([:user_id, :access_level, :expires_at])) - member.valid? # populate the errors - # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) @@ -92,21 +75,17 @@ module API end end - # Update a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # PUT /groups/:id/members/:user_id - # PUT /projects/:id/members/:user_id + desc 'Updates a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end put ":id/members/:user_id" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] member = source.members.find_by!(user_id: params[:user_id]) attrs = attributes_for_keys [:access_level, :expires_at] @@ -121,18 +100,12 @@ module API end end - # Remove a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # DELETE /groups/:id/members/:user_id - # DELETE /projects/:id/members/:user_id + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - required_attributes! [:user_id] # This is to ensure back-compatibility but find_by! should be used # in that casse in 9.0! @@ -147,7 +120,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(member, current_user).execute + ::Members::DestroyService.new(source, current_user, declared(params)).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 2b685621da9..bf8504e1101 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -86,14 +86,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute if merge_request.valid? - # Find or create labels and attach to issue - if params[:labels].present? - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors @@ -195,15 +192,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..9b73f6826cf 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -108,8 +108,7 @@ module API finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: @milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 25dc41a3349..cd398274507 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -4,20 +4,18 @@ module API before { authenticate! } resource :namespaces do - # Get a namespaces list - # - # Example Request: - # GET /namespaces + desc 'Get a namespaces list' do + success Entities::Namespace + end + params do + optional :search, type: String, desc: "Search query for namespaces" + end get do - @namespaces = if current_user.admin? - Namespace.all - else - current_user.namespaces - end - @namespaces = @namespaces.search(params[:search]) if params[:search].present? - @namespaces = paginate @namespaces + namespaces = current_user.admin? ? Namespace.all : current_user.namespaces + + namespaces = namespaces.search(params[:search]) if params[:search].present? - present @namespaces, with: Entities::Namespace + present paginate(namespaces), with: Entities::Namespace end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6d99617b56f..da16e24d7ea 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -22,14 +22,25 @@ module API # Example Request: # GET /projects get do - @projects = current_user.authorized_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - if params[:simple] - present @projects, with: Entities::BasicProjectDetails, user: current_user - else - present @projects, with: Entities::ProjectWithAccess, user: current_user - end + projects = current_user.authorized_projects + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user + end + + # Get a list of visible projects for authenticated user + # + # Example Request: + # GET /projects/visible + get '/visible' do + projects = ProjectsFinder.new.execute(current_user) + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user end # Get an owned projects list for authenticated user @@ -37,10 +48,10 @@ module API # Example Request: # GET /projects/owned get '/owned' do - @projects = current_user.owned_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = current_user.owned_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Gets starred project for the authenticated user @@ -48,10 +59,10 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.viewable_starred_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::Project, user: current_user + projects = current_user.viewable_starred_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::Project, user: current_user end # Get all projects for admin user @@ -60,10 +71,10 @@ module API # GET /projects/all get '/all' do authenticated_as_admin! - @projects = Project.all - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = Project.all + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Get a single project @@ -393,23 +404,30 @@ module API # Share project with group # # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group + # id (required) - The ID of a project + # group_id (required) - The ID of a group # group_access (required) - Level of permissions for sharing + # expires_at (optional) - Share expiration date # # Example Request: # POST /projects/:id/share post ":id/share" do authorize! :admin_project, user_project required_attributes! [:group_id, :group_access] + attrs = attributes_for_keys [:group_id, :group_access, :expires_at] + + group = Group.find_by_id(attrs[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(attrs) + if link.save present link, with: Entities::ProjectGroupLink else diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 22b8f90dc5c..794e34874f4 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -7,38 +7,36 @@ module API end resource :hooks do - # Get the list of system hooks - # - # Example Request: - # GET /hooks + desc 'Get the list of system hooks' do + success Entities::Hook + end get do - @hooks = SystemHook.all - present @hooks, with: Entities::Hook + hooks = SystemHook.all + present hooks, with: Entities::Hook end - # Create new system hook - # - # Parameters: - # url (required) - url for system hook - # Example Request - # POST /hooks + desc 'Create a new system hook' do + success Entities::Hook + end + params do + requires :url, type: String, desc: 'The URL for the system hook' + end post do - attrs = attributes_for_keys [:url] - required_attributes! [:url] - @hook = SystemHook.new attrs - if @hook.save - present @hook, with: Entities::Hook + hook = SystemHook.new declared(params).to_h + + if hook.save + present hook, with: Entities::Hook else not_found! end end - # Test a hook - # - # Example Request - # GET /hooks/:id + desc 'Test a hook' + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end get ":id" do - @hook = SystemHook.find(params[:id]) + hook = SystemHook.find(params[:id]) data = { event_name: "project_create", name: "Ruby", @@ -47,23 +45,21 @@ module API owner_name: "Someone", owner_email: "example@gitlabhq.com" } - @hook.execute(data, 'system_hooks') + hook.execute(data, 'system_hooks') data end - # Delete a hook. This is an idempotent function. - # - # Parameters: - # id (required) - ID of the hook - # Example Request: - # DELETE /hooks/:id + desc 'Delete a hook' do + success Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end delete ":id" do - begin - @hook = SystemHook.find(params[:id]) - @hook.destroy - rescue - # SystemHook raises an Error if no hook with id found - end + hook = SystemHook.find_by(id: params[:id]) + not_found!('System hook') unless hook + + present hook.destroy, with: Entities::Hook end end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index b9e718147e1..8a53d9c0095 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,39 +1,115 @@ module API class Templates < Grape::API GLOBAL_TEMPLATE_TYPES = { - gitignores: Gitlab::Template::GitignoreTemplate, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + } }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + DEPRECATION_MESSAGE = ' This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + def render_response(template_type, template) not_found!(template_type.to_s.singularize) unless template present template, with: Entities::Template end end - GLOBAL_TEMPLATE_TYPES.each do |template_type, klass| - # Get the list of the available template - # - # Example Request: - # GET /gitignores - # GET /gitlab_ci_ymls - get template_type.to_s do - present klass.all, with: Entities::TemplatesList - end - - # Get the text for a specific template present in local filesystem - # - # Parameters: - # name (required) - The name of a template - # - # Example Request: - # GET /gitignores/Elixir - # GET /gitlab_ci_ymls/Ruby - get "#{template_type}/:name" do - required_attributes! [:name] - new_template = klass.find(params[:name]) - render_response(template_type, new_template) + { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| + desc 'Get the list of the available license template' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + end + get route do + options = { + featured: declared(params).popular.present? ? true : nil + } + present Licensee::License.all(options), with: Entities::RepoLicense + end + end + + { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific license' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route, requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: Entities::RepoLicense + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| + desc 'Get the list of the available template' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::TemplatesList + end + get route do + present klass.all, with: Entities::TemplatesList + end + end + + { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific template present in local filesystem' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route do + new_template = klass.find(declared(params).name) + + render_response(template_type, new_template) + end end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 19df13d8aac..832b04a3bb1 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -8,18 +8,19 @@ module API 'issues' => ->(id) { find_project_issue(id) } } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_id".to_sym - # Create a todo on an issuable - # - # Parameters: - # id (required) - The ID of a project - # issuable_id (required) - The ID of an issuable - # Example Request: - # POST /projects/:id/issues/:issuable_id/todo - # POST /projects/:id/merge_requests/:issuable_id/todo + desc 'Create a todo on an issuable' do + success Entities::Todo + end + params do + requires type_id_str, type: Integer, desc: 'The ID of an issuable' + end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) todo = TodoService.new.mark_todo(issuable, current_user).first @@ -40,25 +41,21 @@ module API end end - # Get a todo list - # - # Example Request: - # GET /todos - # + desc 'Get a todo list' do + success Entities::Todo + end get do todos = find_todos present paginate(todos), with: Entities::Todo, current_user: current_user end - # Mark a todo as done - # - # Parameters: - # id: (required) - The ID of the todo being marked as done - # - # Example Request: - # DELETE /todos/:id - # + desc 'Mark a todo as done' do + success Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end delete ':id' do todo = current_user.todos.find(params[:id]) TodoService.new.mark_todos_as_done([todo], current_user) @@ -66,11 +63,7 @@ module API present todo.reload, with: Entities::Todo, current_user: current_user end - # Mark all todos as done - # - # Example Request: - # DELETE /todos - # + desc 'Mark all todos as done' delete do todos = find_todos TodoService.new.mark_todos_as_done(todos, current_user) diff --git a/lib/api/users.rb b/lib/api/users.rb index 4924182770b..6ab9786a97a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -338,6 +338,26 @@ module API user.activate end end + + desc 'Get contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + requires :id, type: String, desc: 'The user ID' + end + get ':id/events' do + user = User.find_by(id: declared(params).id) + not_found!('User') unless user + + events = user.recent_events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + page(params[:page]) + + present paginate(events), with: Entities::Event + end end resource :user do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f6495071a11..b9fb3c21dbb 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -4,27 +4,29 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do - # Get project variables - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/variables + desc 'Get project variables' do + success Entities::Variable + end + params do + optional :page, type: Integer, desc: 'The page number for pagination' + optional :per_page, type: Integer, desc: 'The value of items per page to show' + end get ':id/variables' do variables = user_project.variables present paginate(variables), with: Entities::Variable end - # Get specific variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The `key` of variable - # Example Request: - # GET /projects/:id/variables/:key + desc 'Get a specific variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end get ':id/variables/:key' do key = params[:key] variable = user_project.variables.find_by(key: key.to_s) @@ -34,18 +36,15 @@ module API present variable, with: Entities::Variable end - # Create a new variable in project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The key of variable - # value (required) - The value of variable - # Example Request: - # POST /projects/:id/variables + desc 'Create a new variable in a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end post ':id/variables' do - required_attributes! [:key, :value] - - variable = user_project.variables.create(key: params[:key], value: params[:value]) + variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) if variable.valid? present variable, with: Entities::Variable @@ -54,41 +53,37 @@ module API end end - # Update existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (optional) - The `key` of variable - # value (optional) - New value for `value` field of variable - # Example Request: - # PUT /projects/:id/variables/:key + desc 'Update an existing variable from a project' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end put ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - attrs = attributes_for_keys [:value] - if variable.update(attrs) + if variable.update(value: params[:value]) present variable, with: Entities::Variable else render_validation_error!(variable) end end - # Delete existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The ID of a variable - # Example Request: - # DELETE /projects/:id/variables/:key + desc 'Delete an existing variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end delete ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - variable.destroy - present variable, with: Entities::Variable + present variable.destroy, with: Entities::Variable end end end diff --git a/lib/api/version.rb b/lib/api/version.rb new file mode 100644 index 00000000000..9ba576bd828 --- /dev/null +++ b/lib/api/version.rb @@ -0,0 +1,12 @@ +module API + class Version < Grape::API + before { authenticate! } + + desc 'Get the version information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 8.13.' + end + get '/version' do + { version: Gitlab::VERSION, revision: Gitlab::REVISION } + end + end +end diff --git a/lib/banzai.rb b/lib/banzai.rb index 9ebe379f454..35ca234c1ba 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.render_field(object, field) + Renderer.render_field(object, field) + end + def self.cache_collection_render(texts_and_contexts) Renderer.cache_collection_render(texts_and_contexts) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index affe34394c2..cb213a76a05 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -208,8 +208,12 @@ module Banzai @references_per_project ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = Regexp.union(object_class.reference_pattern, - object_class.link_reference_pattern) + regex = + if uses_reference_pattern? + Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) + else + object_class.link_reference_pattern + end nodes.each do |node| node.to_html.scan(regex) do @@ -295,6 +299,14 @@ module Banzai value end end + + # There might be special cases like filters + # that should ignore reference pattern + # eg: IssueReferenceFilter when using a external issues tracker + # In those cases this method should be overridden on the filter subclass + def uses_reference_pattern? + true + end end end end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 2492b5213ac..a8c1ca0c60a 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,6 +1,6 @@ module Banzai module Filter - # HTML filter that replaces :emoji: with images. + # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter # @@ -13,16 +13,17 @@ module Banzai def call search_text_nodes(doc).each do |node| content = node.to_html - next unless content.include?(':') next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - html = emoji_image_filter(content) + next unless content.include?(':') || node.text.match(emoji_unicode_pattern) + + html = emoji_name_image_filter(content) + html = emoji_unicode_image_filter(html) next if html == content node.replace(html) end - doc end @@ -31,18 +32,38 @@ module Banzai # text - String text to replace :emoji: in. # # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) + def emoji_name_image_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />" + emoji_image_tag(name, emoji_url(name)) end end + # Replace unicode emoji with corresponding images if they exist. + # + # text - String text to replace unicode emoji in. + # + # Returns a String with unicode emoji replaced with images. + def emoji_unicode_image_filter(text) + text.gsub(emoji_unicode_pattern) do |moji| + emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + end + end + + def emoji_image_tag(emoji_name, emoji_url) + "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" + end + # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end + # Build a regexp that matches all valid unicode emojis names. + def self.emoji_unicode_pattern + @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + end + private def emoji_url(name) @@ -60,6 +81,18 @@ module Banzai end end + def emoji_unicode_url(moji) + emoji_unicode_path = emoji_unicode_filename(moji) + + if context[:asset_host] + url_to_image(emoji_unicode_path) + elsif context[:asset_root] + File.join(context[:asset_root], url_to_image(emoji_unicode_path)) + else + url_to_image(emoji_unicode_path) + end + end + def url_to_image(image) ActionController::Base.helpers.url_to_image(image) end @@ -71,6 +104,14 @@ module Banzai def emoji_filename(name) "#{Gitlab::Emoji.emoji_filename(name)}.png" end + + def emoji_unicode_pattern + self.class.emoji_unicode_pattern + end + + def emoji_unicode_filename(name) + "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index eaa702952cc..0d20be557a0 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -8,7 +8,7 @@ module Banzai # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text) do |match, issue| + # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -17,8 +17,8 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(ExternalIssue.reference_pattern) do |match| + def self.references_in(text, pattern) + text.gsub(pattern) do |match| yield match, $~[:issue] end end @@ -27,7 +27,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = ExternalIssue.reference_pattern + ref_pattern = issue_reference_pattern ref_start_pattern = /\A#{ref_pattern}\z/ each_node do |node| @@ -60,7 +60,7 @@ module Banzai def issue_link_filter(text, link_text: nil) project = context[:project] - self.class.references_in(text) do |match, id| + self.class.references_in(text, issue_reference_pattern) do |match, id| ExternalIssue.new(id, project) url = url_for_issue(id, project, only_path: context[:only_path]) @@ -82,18 +82,21 @@ module Banzai end def default_issues_tracker? - if RequestStore.active? - default_issues_tracker_cache[project.id] ||= - project.default_issues_tracker? - else - project.default_issues_tracker? - end + external_issues_cached(:default_issues_tracker?) + end + + def issue_reference_pattern + external_issues_cached(:issue_reference_pattern) end private - def default_issues_tracker_cache - RequestStore[:banzai_default_issues_tracker_cache] ||= {} + def external_issues_cached(attribute) + return project.public_send(attribute) unless RequestStore.active? + + cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 0a29c547a4d..2f19b59e725 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -3,10 +3,17 @@ module Banzai # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - # Skip non-HTTP(S) links and internal links - doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| - node.set_attribute('rel', 'nofollow noreferrer') - node.set_attribute('target', '_blank') + links.each do |node| + href = href_to_lowercase_scheme(node["href"].to_s) + + unless node["href"].to_s == href + node.set_attribute('href', href) + end + + if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') + end end doc @@ -14,6 +21,25 @@ module Banzai private + def links + query = 'descendant-or-self::a[@href and not(@href = "")]' + doc.xpath(query) + end + + def href_to_lowercase_scheme(href) + scheme_match = href.match(/\A(\w+):\/\//) + + if scheme_match + scheme_match.to_s.downcase + scheme_match.post_match + else + href + end + end + + def external_url?(url) + !url.start_with?(internal_url) + end + def internal_url @internal_url ||= Gitlab.config.gitlab.url end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb new file mode 100644 index 00000000000..f3bd587c28b --- /dev/null +++ b/lib/banzai/filter/html_entity_filter.rb @@ -0,0 +1,12 @@ +require 'erb' + +module Banzai + module Filter + # Text filter that escapes these HTML entities: & " < > + class HtmlEntityFilter < HTML::Pipeline::TextFilter + def call + ERB::Util.html_escape_once(text) + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 54c5f9a71a4..4d1bc687696 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -4,6 +4,10 @@ module Banzai # issues that do not exist are ignored. # # This filter supports cross-project references. + # + # When external issues tracker like Jira is activated we should not + # use issue reference pattern, but we should still be able + # to reference issues from other GitLab projects. class IssueReferenceFilter < AbstractReferenceFilter self.reference_type = :issue @@ -11,6 +15,10 @@ module Banzai Issue end + def uses_reference_pattern? + context[:project].default_issues_tracker? + end + def find_object(project, iid) issues_per_project[project][iid] end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8f262ef3d8d..c24831e68ee 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -9,7 +9,7 @@ module Banzai end def find_object(project, id) - project.labels.find(id) + find_labels(project).find(id) end def self.references_in(text, pattern = Label.reference_pattern) @@ -35,7 +35,11 @@ module Banzai return unless project label_params = label_params(label_id, label_name) - project.labels.find_by(label_params) + find_labels(project).find_by(label_params) + end + + def find_labels(project) + LabelsFinder.new(nil, project_id: project.id).execute(authorized_only: false) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -60,13 +64,50 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - LabelsHelper.render_colored_label(object) + if same_group?(object) && namespace_match?(matches) + render_same_project_label(object) + elsif same_project?(object) + render_same_project_label(object) else - LabelsHelper.render_colored_cross_project_label(object) + render_cross_project_label(object, matches) end end + def same_group?(object) + object.is_a?(GroupLabel) && object.group == project.group + end + + def namespace_match?(matches) + matches[:project].blank? || matches[:project] == project.path_with_namespace + end + + def same_project?(object) + object.is_a?(ProjectLabel) && object.project == project + end + + def user + context[:current_user] || context[:author] + end + + def project + context[:project] + end + + def render_same_project_label(object) + LabelsHelper.render_colored_label(object) + end + + def render_cross_project_label(object, matches) + source_project = + if matches[:project] + Project.find_with_namespace(matches[:project]) + else + object.project + end + + LabelsHelper.render_colored_cross_project_label(object, source_project) + end + def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 2470362e019..af1e575fc89 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,7 +25,7 @@ module Banzai return if customized?(whitelist[:transformers]) # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) + whitelist[:attributes]['pre'] = %w(class v-pre) whitelist[:attributes]['span'] = %w(class) # Allow table alignment diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb new file mode 100644 index 00000000000..c2976aeb7c6 --- /dev/null +++ b/lib/banzai/filter/set_direction_filter.rb @@ -0,0 +1,15 @@ +module Banzai + module Filter + # HTML filter that sets dir="auto" for RTL languages support + class SetDirectionFilter < HTML::Pipeline::Filter + def call + # select these elements just on top level of the document + doc.xpath('p|h1|h2|h3|h4|h5|h6|ol|ul[not(@class="section-nav")]|blockquote|table').each do |el| + el['dir'] = 'auto' + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fcdb496aed2..026b81ac175 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -30,7 +30,7 @@ module Banzai # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..9fa5f589f3e 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,27 +2,7 @@ require 'task_list/filter' module Banzai module Filter - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) - if new_class_names.include?('task-list') - # Don't add class to all lists - return - elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') - end - - add_css_class_without_fix(node, *new_class_names) - end - - alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index e1ca7f4d24b..c6302b586d3 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -106,13 +106,17 @@ module Banzai project = context[:project] author = context[:author] - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) + if author && !project.team.member?(author) + link_text + else + url = urls.namespace_project_url(project.namespace, project, + only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + data = data_attribute(project: project.id, author: author.try(:id)) + text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, text, 'All Project and Group Members') + end end def link_to_namespace(namespace, link_text: nil) diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb index bab6a9934d1..2b7c10f1a0e 100644 --- a/lib/banzai/note_renderer.rb +++ b/lib/banzai/note_renderer.rb @@ -3,7 +3,7 @@ module Banzai # Renders a collection of Note instances. # # notes - The notes to render. - # project - The project to use for rendering/redacting. + # project - The project to use for redacting. # user - The user viewing the notes. # path - The request path. # wiki - The project's wiki. @@ -13,8 +13,7 @@ module Banzai user, requested_path: path, project_wiki: wiki, - ref: git_ref, - pipeline: :note) + ref: git_ref) renderer.render(notes, :note) end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9aef807c152..9f8eb0931b8 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -1,28 +1,32 @@ module Banzai - # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # Class for rendering multiple objects (e.g. Note instances) in a single pass, + # using +render_field+ to benefit from caching in the database. Rendering and + # redaction are both performed. # - # Rendered Markdown is stored in an attribute in every object based on the - # name of the attribute containing the Markdown. For example, when the - # attribute `note` is rendered the HTML is stored in `note_html`. + # The unredacted HTML is generated according to the usual +render_field+ + # policy, so specify the pipeline and any other context options on the model. + # + # The *redacted* (i.e., suitable for use) HTML is placed in an attribute + # named "redacted_<foo>", where <foo> is the name of the cache field for the + # chosen attribute. + # + # As an example, rendering the attribute `note` would place the unredacted + # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer attr_reader :project, :user - # Make sure to set the appropriate pipeline in the `raw_context` attribute - # (e.g. `:note` for Note instances). - # - # project - A Project to use for rendering and redacting Markdown. + # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use in the rendering - # pipeline. - def initialize(project, user = nil, raw_context = {}) + # context - A Hash containing extra attributes to use during redaction + def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @raw_context = raw_context + @redaction_context = redaction_context end # Renders and redacts an Array of objects. # - # objects - The objects to render + # objects - The objects to render. # attribute - The attribute containing the raw Markdown to render. # # Returns the same input objects. @@ -32,7 +36,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) object.user_visible_reference_count = redacted_data[:visible_reference_count] end end @@ -53,12 +57,8 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.merge(cache_key: [object, attribute]) - - if object.respond_to?(:author) - context[:author] = object.author - end - + context = base_context.dup + context = context.merge(object.banzai_render_context(attribute)) context end @@ -66,21 +66,16 @@ module Banzai # # Returns an Array of `Nokogiri::HTML::Document`. def render_attributes(objects, attribute) - strings_and_contexts = objects.map do |object| + objects.map do |object| + string = Banzai.render_field(object, attribute) context = context_for(object, attribute) - string = object.__send__(attribute) - - { text: string, context: context } - end - - Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| - Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + Banzai::Pipeline[:relative_link].to_document(string, context) end end def base_context - @base_context ||= @raw_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge(current_user: user, project: project) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8d94b199c66..5da2d0b008c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -25,7 +25,9 @@ module Banzai Filter::MilestoneReferenceFilter, Filter::TaskListFilter, - Filter::InlineDiffFilter + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index ba2555df98d..1929099931b 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,6 +3,7 @@ module Banzai class SingleLinePipeline < GfmPipeline def self.filters @filters ||= FilterArray[ + Filter::HtmlEntityFilter, Filter::SanitizationFilter, Filter::EmojiFilter, diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index a4ae27eefd8..ce048a36fa0 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,6 +1,6 @@ module Banzai module Renderer - extend self + module_function # Convert a Markdown String into an HTML-safe String of HTML # @@ -31,6 +31,34 @@ module Banzai end end + # Convert a Markdown-containing field on an object into an HTML-safe String + # of HTML. This method is analogous to calling render(object.field), but it + # can cache the rendered HTML in the object, rather than Redis. + # + # The context to use is learned from the passed-in object by calling + # #banzai_render_context(field), and cannot be changed. Use #render, passing + # it the field text, if a custom rendering is needed. The generated context + # is returned along with the HTML. + def render_field(object, field) + html_field = object.markdown_cache_field_for(field) + + html = object.__send__(html_field) + return html if html.present? + + html = cacheless_render_field(object, field) + object.update_column(html_field, html) unless object.new_record? || object.destroyed? + + html + end + + # Same as +render_field+, but without consulting or updating the cache field + def cacheless_render_field(object, field) + text = object.__send__(field) + context = object.banzai_render_context(field) + + cacheless_render(text, context) + end + # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # @@ -113,8 +141,6 @@ module Banzai end.html_safe end - private - def cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 59f85416ee5..ed87a2603e8 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,10 +12,9 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact(save: false) - update_runner_info required_attributes! [:token] not_found! unless current_runner.active? + update_runner_info build = Ci::RegisterBuildService.new.execute(current_runner) @@ -41,10 +40,11 @@ module Ci # PUT /builds/:id put ":id" do authenticate_runner! - update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) forbidden!('Build has been erased!') if build.erased? + update_runner_info + build.update_attributes(trace: params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 23353c62885..e608f5f6cad 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 40 * 60 + UPDATE_RUNNER_EVERY = 10 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -30,14 +30,22 @@ module Ci token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact(save: true) - # Use a random threshold to prevent beating DB updates - # it generates a distribution between: [40m, 80m] + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.contacted_at = Time.now - current_runner.save if current_runner.changed? && save - end + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age end def build_not_found! @@ -57,11 +65,6 @@ module Ci attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) end - def update_runner_info - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - def max_artifacts_size current_application_settings.max_artifacts_size.megabytes.to_i end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 0369e80312a..3e33c9399e2 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,7 +4,7 @@ module Ci include Gitlab::Ci::Config::Node::LegacyValidationHelpers - attr_reader :path, :cache, :stages + attr_reader :path, :cache, :stages, :jobs def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -109,6 +109,7 @@ module Ci validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_environment!(name, job) end end @@ -150,6 +151,35 @@ module Ci end end + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb new file mode 100644 index 00000000000..ca39b1961ae --- /dev/null +++ b/lib/constraints/group_url_constrainer.rb @@ -0,0 +1,7 @@ +require 'constraints/namespace_url_constrainer' + +class GroupUrlConstrainer < NamespaceUrlConstrainer + def find_resource(id) + Group.find_by_path(id) + end +end diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb new file mode 100644 index 00000000000..91b70143f11 --- /dev/null +++ b/lib/constraints/namespace_url_constrainer.rb @@ -0,0 +1,24 @@ +class NamespaceUrlConstrainer + def matches?(request) + id = request.path + id = id.sub(/\A#{relative_url_root}/, '') if relative_url_root + id = id.sub(/\A\/+/, '').split('/').first + id = id.sub(/.atom\z/, '') if id + + if id =~ Gitlab::Regex.namespace_regex + find_resource(id) + end + end + + def find_resource(id) + Namespace.find_by_path(id) + end + + private + + def relative_url_root + if defined?(Gitlab::Application.config.relative_url_root) + Gitlab::Application.config.relative_url_root + end + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb new file mode 100644 index 00000000000..504a0f5d93e --- /dev/null +++ b/lib/constraints/user_url_constrainer.rb @@ -0,0 +1,7 @@ +require 'constraints/namespace_url_constrainer' + +class UserUrlConstrainer < NamespaceUrlConstrainer + def find_resource(id) + User.find_by('lower(username) = ?', id.downcase) + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 668d2fa41b3..21f6a9a762b 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -2,8 +2,8 @@ class EventFilter attr_accessor :params class << self - def default_filter - %w{ push issues merge_requests team} + def all + 'all' end def push @@ -35,18 +35,28 @@ class EventFilter return events unless params.present? filter = params.dup - actions = [] - actions << Event::PUSHED if filter.include? 'push' - actions << Event::MERGED if filter.include? 'merged' - if filter.include? 'team' - actions << Event::JOINED - actions << Event::LEFT + case filter + when EventFilter.push + actions = [Event::PUSHED] + when EventFilter.merged + actions = [Event::MERGED] + when EventFilter.comments + actions = [Event::COMMENTED] + when EventFilter.team + actions = [Event::JOINED, Event::LEFT, Event::EXPIRED] + when EventFilter.all + actions = [ + Event::PUSHED, + Event::MERGED, + Event::COMMENTED, + Event::JOINED, + Event::LEFT, + Event::EXPIRED + ] end - actions << Event::COMMENTED if filter.include? 'comments' - events.where(action: actions) end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a4558d157c0..9b74364849e 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -52,8 +52,7 @@ module ExtractsPath # Append a trailing slash if we only get a ref and no file path id += '/' unless id.ends_with?('/') - valid_refs = @project.repository.ref_names - valid_refs.select! { |v| id.start_with?("#{v}/") } + valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } if valid_refs.length == 0 # No exact ref match, so just try our best @@ -74,6 +73,19 @@ module ExtractsPath pair end + # If we have an ID of 'foo.atom', and the controller provides Atom and HTML + # formats, then we have to check if the request was for the Atom version of + # the ID without the '.atom' suffix, or the HTML version of the ID including + # the suffix. We only check this if the version including the suffix doesn't + # match, so it is possible to create a branch which has an unroutable Atom + # feed. + def extract_ref_without_atom(id) + id_without_atom = id.sub(/\.atom$/, '') + valid_refs = ref_names.select { |v| "#{id_without_atom}/".start_with?("#{v}/") } + + valid_refs.max_by(&:length) + end + # Assigns common instance variables for views working with Git tree-ish objects # # Assignments are: @@ -86,6 +98,10 @@ module ExtractsPath # If the :id parameter appears to be requesting a specific response format, # that will be handled as well. # + # If there is no path and the ref doesn't exist in the repo, try to resolve + # the ref without an '.atom' suffix. If _that_ ref is found, set the request's + # format to Atom manually. + # # Automatically renders `not_found!` if a valid tree path could not be # resolved (e.g., when a user inserts an invalid path or ref). def assign_ref_vars @@ -97,10 +113,18 @@ module ExtractsPath @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository - if @options[:extended_sha1].blank? - @commit = @repo.commit(@ref) - else + + if @options[:extended_sha1].present? @commit = @repo.commit(@options[:extended_sha1]) + else + @commit = @repo.commit(@ref) + + if @path.empty? && !@commit && @id.ends_with?('.atom') + @id = @ref = extract_ref_without_atom(@id) + @commit = @repo.commit(@ref) + + request.format = :atom if @commit + end end raise InvalidPathError unless @commit @@ -125,4 +149,10 @@ module ExtractsPath id += "/" + params[:path] unless params[:path].blank? id end + + def ref_names + return [] unless @project + + @ref_names ||= @project.repository.ref_names + end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index a533bac2692..9b484a2ecfd 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -53,6 +53,10 @@ module Gitlab } end + def sym_options_with_owner + sym_options.merge(owner: OWNER) + end + def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 79eac66b364..9cec71a3222 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -17,6 +17,18 @@ module Gitlab end class << self + def secret_token + @secret_token ||= begin + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + end + + def ensure_secret_token! + return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')) + + generate_and_link_secret_token + end + def version_required @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip @@ -25,6 +37,25 @@ module Gitlab def strip_key(key) key.split(/ /)[0, 2].join(' ') end + + private + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + shell_path = Gitlab.config.gitlab_shell.path + + unless File.size?(secret_file) + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + @secret_token = SecureRandom.hex(16) + File.write(secret_file, @secret_token) + end + + link_path = File.join(shell_path, '.gitlab_shell_secret') + if File.exist?(shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end + end end # Init new repository @@ -201,21 +232,6 @@ module Gitlab File.exist?(full_path(storage, dir_name)) end - # Create (if necessary) and link the secret token file - def generate_and_link_secret_token - secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.size?(secret_file) - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) - end - - link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') - if File.exist?(gitlab_shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) - end - end - protected def gitlab_shell_path diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb index d388ab6b879..9a95ef43628 100644 --- a/lib/gitlab/ci/config/node/environment.rb +++ b/lib/gitlab/ci/config/node/environment.rb @@ -8,7 +8,7 @@ module Gitlab class Environment < Entry include Validatable - ALLOWED_KEYS = %i[name url] + ALLOWED_KEYS = %i[name url action on_stop] validations do validate do @@ -35,6 +35,12 @@ module Gitlab length: { maximum: 255 }, addressable_url: true, allow_nil: true + + validates :action, + inclusion: { in: %w[start stop], message: 'should be start or stop' }, + allow_nil: true + + validates :on_stop, type: String, allow_nil: true end end @@ -54,9 +60,17 @@ module Gitlab value[:url] end + def action + value[:action] || 'start' + end + + def on_stop + value[:on_stop] + end + def value case @config - when String then { name: @config } + when String then { name: @config, action: 'start' } when Hash then @config else {} end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb new file mode 100644 index 00000000000..37e51536e8f --- /dev/null +++ b/lib/gitlab/ci/trace_reader.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class TraceReader + BUFFER_SIZE = 4096 + + attr_accessor :path, :buffer_size + + def initialize(new_path, buffer_size: BUFFER_SIZE) + self.path = new_path + self.buffer_size = Integer(buffer_size) + end + + def read(last_lines: nil) + if last_lines + read_last_lines(last_lines) + else + File.read(path) + end + end + + def read_last_lines(max_lines) + File.open(path) do |file| + chunks = [] + pos = lines = 0 + max = file.size + + # We want an extra line to make sure fist line has full contents + while lines <= max_lines && pos < max + pos += buffer_size + + buf = if pos <= max + file.seek(-pos, IO::SEEK_END) + file.read(buffer_size) + else # Reached the head, read only left + file.seek(0) + file.read(buffer_size - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(max_lines).join + end + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index dff9e29c6a5..c843315782d 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,7 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers include IconsHelper - class MissingResolution < StandardError + class MissingResolution < ResolutionError end CONTEXT_LINES = 3 @@ -21,12 +21,34 @@ module Gitlab @match_line_headers = {} end + def content + merge_file_result[:data] + end + + def our_blob + @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) + end + + def type + lines unless @type + + @type.inquiry + end + # Array of Gitlab::Diff::Line objects def lines - @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data], + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Conflict::Parser.new.parse(content, our_path: our_path, their_path: their_path, parent_file: self) + rescue Gitlab::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end end def resolve_lines(resolution) @@ -53,6 +75,14 @@ module Gitlab end.compact end + def resolve_content(resolution) + if resolution == content + raise MissingResolution, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + def highlight_lines! their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") @@ -170,21 +200,39 @@ module Gitlab match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" end - def as_json(opts = nil) - { + def as_json(opts = {}) + json_hash = { old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), blob_path: namespace_project_blob_path(merge_request.project.namespace, merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)), - sections: sections + ::File.join(merge_request.diff_refs.head_sha, our_path)) } + + json_hash.tap do |json_hash| + if opts[:full_content] + json_hash[:content] = content + json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode) + else + json_hash[:sections] = sections if type.text? + json_hash[:type] = type + json_hash[:content_path] = content_path + end + end + end + + def content_path + conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, + merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. def inspect - instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode].map do |instance_variable| + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| value = instance_variable_get("@#{instance_variable}") "#{instance_variable}=\"#{value}\"" diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index bbd0427a2c8..fa5bd4649d4 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -30,6 +30,10 @@ module Gitlab end end + def file_for_path(old_path, new_path) + files.find { |file| file.their_path == old_path && file.our_path == new_path } + end + def as_json(opts = nil) { target_branch: merge_request.target_branch, diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 98e842cded3..ddd657903fb 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -1,19 +1,24 @@ module Gitlab module Conflict class Parser - class ParserError < StandardError + class UnresolvableError < StandardError end - class UnexpectedDelimiter < ParserError + class UnmergeableFile < UnresolvableError end - class MissingEndDelimiter < ParserError + class UnsupportedEncoding < UnresolvableError + end + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + class ParserError < StandardError end - class UnmergeableFile < ParserError + class UnexpectedDelimiter < ParserError end - class UnsupportedEncoding < ParserError + class MissingEndDelimiter < ParserError end def parse(text, our_path:, their_path:, parent_file: nil) diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb new file mode 100644 index 00000000000..a0f2006bc24 --- /dev/null +++ b/lib/gitlab/conflict/resolution_error.rb @@ -0,0 +1,6 @@ +module Gitlab + module Conflict + class ResolutionError < StandardError + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e47df508ca2..ce85e5e0123 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -125,6 +125,10 @@ module Gitlab repository.blob_at(commit.id, file_path) end + + def cache_key + "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 36348b33943..dc4d47c878b 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -35,16 +35,16 @@ module Gitlab # for the highlighted ones, so we just skip their execution. # If the highlighted diff files lines are not cached we calculate and cache them. # - # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of + # The content of the cache is a Hash where the key identifies the file and the values are Arrays of # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - file_path = diff_file.file_path + item_key = diff_file.cache_key - if highlight_cache[file_path] - highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path]) + if highlight_cache[item_key] + highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) else - highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 06dae31cc27..447c7a6a6b9 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -46,7 +46,9 @@ module Gitlab noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code + line_code: sent_notification.line_code, + position: sent_notification.position, + type: sent_notification.note_type ).execute end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index b63213ae208..bbbca8acc40 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -10,12 +10,20 @@ module Gitlab Gemojione.index.instance_variable_get(:@emoji_by_moji) end + def emojis_unicodes + emojis_by_moji.keys + end + def emojis_names - emojis.keys.sort + emojis.keys end def emoji_filename(name) emojis[name]["unicode"] end + + def emoji_unicode_filename(moji) + emojis_by_moji[moji]["unicode"] + end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 501d5a95547..65ee85ca5a9 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -74,8 +74,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, title: name, color: color) + params = { title: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end def user_info(person_id) @@ -122,25 +122,21 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed' + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_id: assignee_id, + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue.add_labels_by_names(labels) - if issue.iid != bug['ixBug'] - issue.update_attribute(:iid, bug['ixBug']) - end + issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) - - issue.update_attribute(:created_at, date) - - last_update = DateTime.parse(bug['dtLastUpdated']) - issue.update_attribute(:updated_at, last_update) end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 78d7a4f27cf..a7c596dced0 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -58,7 +58,7 @@ module Gitlab referable = find_referable(reference) return reference unless referable - cross_reference = referable.to_reference(target_project) + cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference new_text = before + cross_reference + after @@ -72,6 +72,14 @@ module Gitlab extractor.all.first end + def build_cross_reference(referable, target_project) + if referable.respond_to?(:project) + referable.to_reference(target_project) + else + referable.to_reference(@source_project, target_project) + end + end + def substitution_valid?(substituted) @original_html == markdown(substituted) end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index e33ac61f5ae..7f424b74efb 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -102,9 +102,19 @@ module Gitlab def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = api.send(method, *args, &block) - yield data + data = api.send(method, *args) + return data unless data.is_a?(Array) + if block_given? + yield data + each_response_page(&block) + else + each_response_page { |page| data.concat(page) } + data + end + end + + def each_response_page last_response = api.last_response while last_response.rels[:next] diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b8321244473..4b70f33a851 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -170,10 +170,9 @@ module Gitlab end def import_wiki - unless project.wiki_enabled? + unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 2cad7fca88e..942dfb3312b 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -14,9 +14,13 @@ module Gitlab end def create! - project.labels.find_or_create_by!(title: title) do |label| - label.color = color - end + params = attributes.except(:project) + service = ::Labels::FindOrCreateService.new(project.owner, project, params) + label = service.execute + + raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + + label end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 605abfabdab..a2410068845 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,7 +1,7 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data def initialize(repo, name, namespace, current_user, session_data) @repo = repo @@ -12,24 +12,37 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, - name: @name, - path: @name, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, + visibility_level: visibility_level, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: import_url, + skip_wiki: skip_wiki ).execute + end + + private - # If repo has wiki we'll import it later - if repo.has_wiki? && project - project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) - end + def import_url + repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + end - project + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 62da327931f..6a68e786b4f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -92,19 +92,17 @@ module Gitlab end issue = Issue.create!( - project_id: project.id, - title: raw_issue["title"], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue["state"] == "closed" ? "closed" : "opened" + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_id: assignee_id, + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue.add_labels_by_names(labels) - - if issue.iid != raw_issue["id"] - issue.update_attribute(:iid, raw_issue["id"]) - end + issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) end @@ -236,8 +234,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, name: name, color: color) + params = { name: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end def format_content(raw_content) diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3e5d728f3bc..f8809db21aa 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,19 +5,61 @@ module Gitlab def identify(identifier, project, newrev) if identifier.blank? # Local push from gitlab - email = project.commit(newrev).author_email rescue nil - User.find_by(email: email) if email - + identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http - user_id = identifier.gsub("user-", "") - User.find_by(id: user_id) - + identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh - key_id = identifier.gsub("key-", "") - Key.find_by(id: key_id).try(:user) + identify_using_ssh_key(identifier) + end + end + + # Tries to identify a user based on a commit SHA. + def identify_using_commit(project, ref) + commit = project.commit(ref) + + return if !commit || !commit.author_email + + email = commit.author_email + + identify_with_cache(:email, email) do + User.find_by(email: email) end end + + # Tries to identify a user based on a user identifier (e.g. "user-123"). + def identify_using_user(identifier) + user_id = identifier.gsub("user-", "") + + identify_with_cache(:user, user_id) do + User.find_by(id: user_id) + end + end + + # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). + def identify_using_ssh_key(identifier) + key_id = identifier.gsub("key-", "") + + identify_with_cache(:ssh_key, key_id) do + User.find_by_ssh_key_id(key_id) + end + end + + def identify_with_cache(category, key) + if identification_cache[category].key?(key) + identification_cache[category][key] + else + identification_cache[category][key] = yield + end + end + + def identification_cache + @identification_cache ||= { + email: {}, + user: {}, + ssh_key: {} + } + end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 181e288a014..eb667a85b78 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.4' + VERSION = '0.1.5' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb new file mode 100644 index 00000000000..f755a404693 --- /dev/null +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -0,0 +1,13 @@ +module Gitlab + module ImportExport + class AttributeCleaner + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + + def self.clean!(relation_hash:) + relation_hash.reject! do |key, _value| + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + end + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e522a0fc8f6..f00c7460e82 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport module CommandLineUtil + DEFAULT_MODE = 0700 + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -21,6 +23,11 @@ module Gitlab execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) end + def mkdir_p(path) + FileUtils.mkdir_p(path, mode: DEFAULT_MODE) + FileUtils.chmod(DEFAULT_MODE, path) + end + private def tar_with_options(archive:, dir:, options:) @@ -45,7 +52,7 @@ module Gitlab # if we are copying files, create the destination folder destination_folder = File.file?(source) ? File.dirname(destination) : destination - FileUtils.mkdir_p(destination_folder) + mkdir_p(destination_folder) FileUtils.copy_entry(source, destination) true end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index eca6e5b6d51..113895ba22c 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -15,7 +15,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) wait_for_archived_file do decompress_archive diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index bb9d1080330..e6ecd118609 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,6 +1,7 @@ # Model relationships to be included in the project import/export project_tree: - - :labels + - labels: + :priorities - milestones: - :events - issues: @@ -9,7 +10,8 @@ project_tree: - :author - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - snippets: @@ -26,7 +28,8 @@ project_tree: - :merge_request_diff - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - pipelines: @@ -71,6 +74,10 @@ excluded_attributes: - :awardable_id methods: + labels: + - :type + label: + - :type statuses: - :type services: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 0cc10f40087..48c09dafcb6 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -65,11 +65,17 @@ module Gitlab # +value+ existing model to be included in the hash # +parsed_hash+ the original hash def parse_hash(value) + return nil if already_contains_methods?(value) + @attributes_finder.parse(value) do |hash| { include: hash_or_merge(value, hash) } end end + def already_contains_methods?(value) + value.is_a?(Hash) && value.values.detect { |val| val[:methods]} + end + # Adds new model configuration to an existing hash with key +current_key+ # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ # diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 35ff134ea19..7cdba880a93 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -110,13 +110,18 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => restored_project.id), + relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, - user: @user) + user: @user, + project_id: restored_project.id) end relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end + + def parsed_relation_hash(relation_hash) + relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + end end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 9153088e966..2fbf437ec26 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport class ProjectTreeSaver + include Gitlab::ImportExport::CommandLineUtil + attr_reader :full_path def initialize(project:, shared:) @@ -10,7 +12,7 @@ module Gitlab end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0c2cd208bf9..ce31c923feb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -9,25 +9,28 @@ module Gitlab builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze + push_access_levels: 'ProtectedBranch::PushAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze - - FINDER_ATTRIBUTES = %w[title project_id].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze def self.create(*args) new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id') + @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -54,6 +57,8 @@ module Gitlab update_user_references update_project_references + + handle_group_label if group_label? reset_ci_tokens if @relation_name == 'Ci::Trigger' @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff @@ -121,6 +126,20 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end + def group_label? + @relation_hash['type'] == 'GroupLabel' + end + + def handle_group_label + # If there's no group, move the label to a project label + if @relation_hash['group_id'] + @relation_hash['project_id'] = nil + @relation_name = :group_label + else + @relation_hash['type'] = 'ProjectLabel' + end + end + def reset_ci_tokens return unless Gitlab::ImportExport.reset_tokens? @@ -153,7 +172,11 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= begin + Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash) + + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end end def set_st_diffs @@ -165,11 +188,9 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - events = parsed_relation_hash.delete('events') + attribute_hash = attribute_hash_for(['events', 'priorities']) - unless events.blank? - existing_object.assign_attributes(events: events) - end + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? existing_object else @@ -178,14 +199,22 @@ module Gitlab end end + def attribute_hash_for(attributes) + attributes.inject({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + def existing_object @existing_object ||= begin - finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES) + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) existing_object = relation_class.find_or_create_by(finder_hash) # Done in two steps, as MySQL behaves differently than PostgreSQL using # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update(parsed_relation_hash) + existing_object.update!(parsed_relation_hash) existing_object end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d1e33ea8678..48a9a6fa5e2 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -12,7 +12,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - FileUtils.mkdir_p(path_to_repo) + mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks rescue => e diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 331e14021e6..a7028a32570 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -20,7 +20,7 @@ module Gitlab private def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 9b642d740b7..7cf88298642 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,12 +1,14 @@ module Gitlab module ImportExport class VersionSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(shared:) @shared = shared end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 6107420e4dd..1e6722a7bba 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -9,7 +9,7 @@ module Gitlab end def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 1bec6088292..dbc759367eb 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -18,8 +18,8 @@ module Gitlab { title: "enhancement", color: green } ] - labels.each do |label| - project.labels.create(label) + labels.each do |params| + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 7b3bbcf6a32..5f67e97fa2a 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -20,13 +20,8 @@ module Gitlab def token Gitlab::Redis.with do |redis| token = redis.get(redis_key) - - if token - redis.expire(redis_key, EXPIRY_TIME) - else - token = Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) - end + token ||= Devise.friendly_token(TOKEN_LENGTH) + redis.set(redis_key, token, ex: EXPIRY_TIME) token end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 5b9cfaeb2f8..24733435a5a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -73,11 +73,7 @@ module Gitlab end def commits - if project.empty_repo? || query.blank? - [] - else - project.repository.find_commits_by_message(query).compact - end + project.repository.find_commits_by_message(query) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 69c4ef721d5..c649da8c426 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -11,13 +11,6 @@ module Gitlab DEFAULT_REDIS_URL = 'redis://localhost:6379' CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) - # To be thread-safe we must be careful when writing the class instance - # variables @_raw_config and @pool. Because @pool depends on @_raw_config we need two - # mutexes to prevent deadlock. - RAW_CONFIG_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :RAW_CONFIG_MUTEX, :POOL_MUTEX - class << self # Do NOT cache in an instance variable. Result may be mutated by caller. def params @@ -31,24 +24,29 @@ module Gitlab end def with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(params) } - end - end + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @pool.with { |redis| yield redis } end + def pool_size + if Sidekiq.server? + # the pool will be used in a multi-threaded context + Sidekiq.options[:concurrency] + 5 + else + # probably this is a Unicorn process, so single threaded + 5 + end + end + def _raw_config return @_raw_config if defined?(@_raw_config) - RAW_CONFIG_MUTEX.synchronize do - begin - @_raw_config = File.read(CONFIG_FILE).freeze - rescue Errno::ENOENT - @_raw_config = false - end + begin + @_raw_config = File.read(CONFIG_FILE).freeze + rescue Errno::ENOENT + @_raw_config = false end + @_raw_config end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 776bbcbb5d0..0d30e1bb92e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,7 +2,7 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze + NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze @@ -10,7 +10,7 @@ module Gitlab def namespace_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.'." \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ end def namespace_name_regex diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 7813091ec7b..82a59a7a87e 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -2,7 +2,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{job['args']}" + Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" yield end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5d33f98e89e..594439a5d4b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -111,7 +111,7 @@ module Gitlab def write_secret bytes = SecureRandom.random_bytes(SECRET_LENGTH) File.open(secret_path, 'w:BINARY', 0600) do |f| - f.chmod(0600) + f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. f.write(Base64.strict_encode64(bytes)) end end diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/.gitkeep +++ /dev/null diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 2214f855200..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,33 @@ namespace :cache do - CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + namespace :clear do + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task :clear => :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: REDIS_CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end + + desc "GitLab | Clear database cache (in the background)" + task db: :environment do + ClearDatabaseCacheWorker.perform_async + end + + task all: [:db, :redis] end + + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake new file mode 100644 index 00000000000..424e7883060 --- /dev/null +++ b/lib/tasks/ce_to_ee_merge_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ce_to_ee_merge_check: :environment do + Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke +end diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/ci/.gitkeep +++ /dev/null diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 5f4a6bbfa35..2ae48a970ce 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -671,7 +671,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -690,7 +690,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -747,7 +747,7 @@ namespace :gitlab do "Check that the information in config/gitlab.yml is correct" ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake new file mode 100644 index 00000000000..47bdb2d32d2 --- /dev/null +++ b/lib/tasks/gitlab/dev.rake @@ -0,0 +1,107 @@ +namespace :gitlab do + namespace :dev do + desc 'Checks if the branch would apply cleanly to EE' + task ce_to_ee_merge_check: :environment do + return if defined?(Gitlab::License) + return unless ENV['CI'] + + ce_repo = ENV['CI_BUILD_REPO'] + ce_branch = ENV['CI_BUILD_REF_NAME'] + + ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' + ee_branch = "#{ce_branch}-ee" + ee_dir = 'gitlab-ee-merge-check' + + puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" + `git clone #{ee_repo} #{ee_dir} --depth 1` + Dir.chdir(ee_dir) do + puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" + `git fetch #{ce_repo} #{ce_branch} --depth 1` + + # Try to merge the current tested branch to EE/master... + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge FETCH_HEAD` + + exit 0 if $?.success? + + # Check if the <branch>-ee branch exists... + puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" + `git rev-parse --verify #{ee_branch}` + + # The <branch>-ee doesn't exist + unless $?.success? + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to the + current EE/master, and no #{ee_branch} branch was detected in + the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + You can create this branch as follows: + + 1. In the EE repo: + $ git fetch origin + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git rebase origin/master + 2. At this point you will likely have conflicts, solve them, and + continue/finish the rebase. Note: You can squash the CE commits + before rebasing. + 3. You can squash all the original #{ce_branch} commits into a + single "Port of #{ce_branch} to EE". + 4. Push your branch to #{ee_repo}: + $ git push origin #{ee_branch} + =================================================================\n + MSG + + exit 1 + end + + # Try to merge the <branch>-ee branch to EE/master... + puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" + `git merge #{ee_branch} master` + + # The <branch>-ee cannot be merged cleanly to EE/master... + unless $?.success? + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to + EE/master, and even though the #{ee_branch} branch exists in the EE + repository, it cannot be merged without conflicts to EE/master. + + Please update the #{ee_branch}, push it again to #{ee_repo}, and + retry this job. + =================================================================\n + MSG + + exit 2 + end + + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge FETCH_HEAD` + exit 0 if $?.success? + + # The <branch>-ee can be merged cleanly to EE/master, but <branch> still + # cannot be merged cleanly to EE/master... + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to EE, and + even though the #{ee_branch} branch exists in the EE repository and + applies cleanly to EE/master, it doesn't prevent conflicts when + merging #{ce_branch} into EE. + + We may be in a complex situation here. + =================================================================\n + MSG + + exit 3 + end + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index bb7eb852f1b..210899882b4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -78,7 +78,7 @@ namespace :gitlab do f.puts "PATH=#{ENV['PATH']}" end - Gitlab::Shell.new.generate_and_link_secret_token + Gitlab::Shell.ensure_secret_token! end desc "GitLab | Setup gitlab-shell" diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake new file mode 100644 index 00000000000..3a16ace60bd --- /dev/null +++ b/lib/tasks/gitlab/users.rake @@ -0,0 +1,11 @@ +namespace :gitlab do + namespace :users do + desc "GitLab | Clear the authentication token for all users" + task clear_all_authentication_tokens: :environment do |t, args| + # Do small batched updates because these updates will be slow and locking + User.select(:id).find_in_batches(batch_size: 100) do |batch| + User.where(id: batch.map(&:id)).update_all(authentication_token: nil) + end + end + end +end |