diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2016-10-25 02:13:24 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2016-10-25 02:13:24 +0800 |
| commit | 600da9ee0bb823e4b14fd45d6ff0e5f0b61b9737 (patch) | |
| tree | f91eeeee22da72eed5bdf5b87ca27bdc95b136a0 /lib | |
| parent | 40ff7579e9ba025610dfada9703386b4dc657d6d (diff) | |
| parent | cb38290ababe43aca0c635fb87d3a38c4c5debcd (diff) | |
| download | gitlab-ce-19737-read-only-auditor.tar.gz | |
Merge remote-tracking branch 'upstream/master' into 19737-read-only-auditor19737-read-only-auditor
* upstream/master: (1277 commits)
Grapify the labels API
Fix typo in project settings that prevents users from enabling container registry.
Fix old monitoring links to point to the new location
Added path parameter to Commits API
fixes build with cache:clear issue
Merge branch 'security-fix-leaking-namespace-name' into 'security'
Fix authored vote from notes
Grapify builds API
Add changelog item for groups 404 on relative url
Add relative url support to routing contrainers
Update project member controller to match recent master logic
Add parentheses around return redirect_to method
Trigger change even in select2 test helper to produce production-like behaviour
Refactor js that disable form submit if no members selected
Improve create project member test at project_members_controller_spec
Move changelog item to 8.14
Refactor create member tests from group_members_controller_spec
Refactor groups/projects members controller
Gracefully handle adding of no users to projects and groups
Revert "Change "Group#web_url" to return "/groups/twitter" rather than "/twitter"."
...
Diffstat (limited to 'lib')
102 files changed, 1851 insertions, 1043 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 9d1d9058996..87915b19480 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -5,32 +5,27 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project access requests viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # GET /groups/:id/access_requests - # GET /projects/:id/access_requests + desc "Gets a list of access requests for a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end get ":id/access_requests" do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - access_requesters = paginate(source.requesters.includes(:user)) + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end - # Request access to the group/project - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # POST /groups/:id/access_requests - # POST /projects/:id/access_requests + desc "Requests access for the authenticated user to a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end post ":id/access_requests" do source = find_source(source_type, params[:id]) access_requester = source.request_access(current_user) @@ -42,42 +37,34 @@ module API end end - # Approve a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # access_level (optional) - Access level - # - # Example Request: - # PUT /groups/:id/access_requests/:user_id/approve - # PUT /projects/:id/access_requests/:user_id/approve + desc 'Approves an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + end put ':id/access_requests/:user_id/approve' do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute status :created present member.user, with: Entities::Member, member: member end - # Deny a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # - # Example Request: - # DELETE /groups/:id/access_requests/:user_id - # DELETE /projects/:id/access_requests/:user_id + desc 'Denies an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + end delete ":id/access_requests/:user_id" do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - access_requester = source.requesters.find_by!(user_id: params[:user_id]) - - ::Members::DestroyService.new(access_requester, current_user).execute + ::Members::DestroyService.new(source, current_user, params). + execute(:requesters) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index cb47ec8f33f..67109ceeef9 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -31,11 +31,12 @@ module API # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji + mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages mount ::API::Builds - mount ::API::CommitStatuses mount ::API::Commits + mount ::API::CommitStatuses mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -45,19 +46,18 @@ module API mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::LicenseTemplates mount ::API::Lint mount ::API::Members - mount ::API::MergeRequests mount ::API::MergeRequestDiffs + mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectSnippets mount ::API::Projects + mount ::API::ProjectSnippets mount ::API::Repositories mount ::API::Runners mount ::API::Services @@ -72,5 +72,10 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::Version + + route :any, '*path' do + error!('404 Not Found', 404) + end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2461a783ea8..e9ccba3b465 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -8,16 +8,19 @@ module API awardable_string = awardable_type.pluralize awardable_id_string = "#{awardable_type}_id" + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) @@ -27,14 +30,13 @@ module API end end - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end get "#{endpoint}/:award_id" do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji @@ -43,17 +45,14 @@ module API end end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/award_emoji + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end post endpoint do - required_attributes! [:name] - not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -65,14 +64,13 @@ module API end end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) diff --git a/lib/api/boards.rb b/lib/api/boards.rb new file mode 100644 index 00000000000..4ac491edc1b --- /dev/null +++ b/lib/api/boards.rb @@ -0,0 +1,132 @@ +module API + # Boards API + class Boards < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + get ':id/boards' do + authorize!(:read_board, user_project) + present user_project.boards, with: Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `backlog` and `done` lists. This feature was introduced in 8.13' + success Entities::List + end + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: Entities::List + end + + desc 'Get a list of a project board' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + end + get '/lists/:list_id' do + authorize!(:read_board, user_project) + present board_lists.find(params[:list_id]), with: Entities::List + end + + desc 'Create a new board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end + post '/lists' do + unless available_labels.exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::CreateService.new(user_project, current_user, + { label_id: params[:label_id] }) + + list = service.execute(project_board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + desc 'Moves a board list to a new position' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + requires :position, type: Integer, desc: 'The position of the list' + end + put '/lists/:list_id' do + list = project_board.lists.movable.find(params[:list_id]) + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::MoveService.new(user_project, current_user, + { position: params[:position] }) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_project) + + list = board_lists.find(params[:list_id]) + + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b615703df93..6d827448994 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -54,43 +54,25 @@ module API not_found!('Branch') unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) - protected_branch_params = { - name: @branch.name + name: @branch.name, + developers_can_push: to_boolean(params[:developers_can_push]), + developers_can_merge: to_boolean(params[:developers_can_merge]) } - # If `developers_can_merge` is switched off, _all_ `DEVELOPER` - # merge_access_levels need to be deleted. - if developers_can_merge == false - protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + service_args = [user_project, current_user, protected_branch_params] - # If `developers_can_push` is switched off, _all_ `DEVELOPER` - # push_access_levels need to be deleted. - if developers_can_push == false - protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - protected_branch_params.merge!( - merge_access_levels_attributes: [{ - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }], - push_access_levels_attributes: [{ - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }] - ) - - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present @branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end # Unprotect a single branch @@ -123,7 +105,7 @@ module API post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], @@ -142,10 +124,10 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 52bdbcae5a8..67adca6605f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -3,15 +3,32 @@ module API class Builds < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ['pending', 'running', 'failed', 'success', 'canceled'], + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +37,13 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +57,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +72,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +86,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +103,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +123,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +140,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +158,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +175,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -235,14 +239,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index dfbdd597d29..f54d4f06627 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -6,17 +6,17 @@ module API resource :projects do before { authenticate! } - # Get a commit's statuses - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # stage (optional) - The stage - # name (optional) - The name - # all (optional) - Show all statuses, default: false - # Examples: - # GET /projects/:id/repository/commits/:sha/statuses + desc "Get a commit's statuses" do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + optional :ref, type: String, desc: 'The ref' + optional :stage, type: String, desc: 'The stage' + optional :name, type: String, desc: 'The name' + optional :all, type: String, desc: 'Show all statuses, default: false' + end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -31,22 +31,23 @@ module API present paginate(statuses), with: Entities::CommitStatus end - # Post status to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, failed or canceled - # target_url (optional) - The target URL to associate with this status - # description (optional) - A short description of the status - # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" - # Examples: - # POST /projects/:id/statuses/:sha + desc 'Post status to a commit' do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + requires :state, type: String, desc: 'The state of the status', + values: ['pending', 'running', 'success', 'failed', 'canceled'] + optional :ref, type: String, desc: 'The ref' + optional :target_url, type: String, desc: 'The target URL to associate with this status' + optional :description, type: String, desc: 'A short description of the status' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + end post ':id/statuses/:sha' do authorize! :create_commit_status, user_project - required_attributes! [:state] - attrs = attributes_for_keys [:target_url, :description] + commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -66,9 +67,14 @@ module API pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) status = GenericCommitStatus.running_or_pending.find_or_initialize_by( - project: @project, pipeline: pipeline, - user: current_user, name: name, ref: ref) - status.attributes = attrs + project: @project, + pipeline: pipeline, + user: current_user, + name: name, + ref: ref, + target_url: params[:target_url], + description: params[:description] + ) begin case params[:state].to_s diff --git a/lib/api/commits.rb b/lib/api/commits.rb index b4eaf1813d4..2f2cf769481 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -6,102 +6,149 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository commits - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # since (optional) - Only commits after or in this date will be returned - # until (optional) - Only commits before or in this date will be returned - # Example Request: - # GET /projects/:id/repository/commits + desc 'Get a project repository commits' do + success Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: String, desc: 'Only commits after or in this date will be returned' + optional :until, type: String, desc: 'Only commits before or in this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end get ":id/repository/commits" do + # TODO remove the next line for 9.0, use DateTime type in the params block datetime_attributes! :since, :until - page = (params[:page] || 0).to_i - per_page = (params[:per_page] || 20).to_i ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - after = params[:since] - before = params[:until] + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) - commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before) present commits, with: Entities::RepoCommit end - # Get a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash or name of a repository branch or tag - # Example Request: - # GET /projects/:id/repository/commits/:sha + desc 'Commit multiple file changes as one commit' do + success Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :id, type: Integer, desc: 'The project ID' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array, desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared(params) + attrs[:source_branch] = attrs[:branch_name] + attrs[:target_branch] = attrs[:branch_name] + attrs[:actions].map! do |action| + action[:action] = action[:action].to_sym + action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/') + action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/') + action + end + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + present commit, with: Entities::RepoCommitDetail end - # Get the diff for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # Example Request: - # GET /projects/:id/repository/commits/:sha/diff + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha/diff" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + commit.raw_diffs.to_a end - # Get a commit's comments - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # Examples: - # GET /projects/:id/repository/commits/:sha/comments + desc "Get a commit's comments" do + success Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :per_page, type: Integer, desc: 'The amount of items per page for paginaion' + optional :page, type: Integer, desc: 'The page number for pagination' + end get ':id/repository/commits/:sha/comments' do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit notes = Note.where(commit_id: commit.id).order(:created_at) + present paginate(notes), with: Entities::CommitNote end - # Post comment to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # note (required) - Text of comment - # path (optional) - The file path - # line (optional) - The line number - # line_type (optional) - The type of line (new or old) - # Examples: - # POST /projects/:id/repository/commits/:sha/comments + desc 'Post comment to commit' do + success Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: ['new', 'old'], default: 'new', desc: 'The type of the line' + end + end post ':id/repository/commits/:sha/comments' do - required_attributes! [:note] - - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit + opts = { note: params[:note], noteable_type: 'Commit', commit_id: commit.id } - if params[:path] && params[:line] && params[:line_type] + if params[:path] commit.raw_diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| - next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + next unless line.new_pos == params[:line] && line.type == params[:line_type] break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b081cb8e520..e1995aa6f4b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -344,7 +344,7 @@ module API end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -433,8 +433,11 @@ module API end end - class Label < Grape::Entity + class LabelBasic < Grape::Entity expose :name, :color, :description + end + + class Label < LabelBasic expose :open_issues_count, :closed_issues_count, :open_merge_requests_count expose :subscribed do |label, options| @@ -442,6 +445,19 @@ module API end end + class List < Grape::Entity + expose :id + expose :label, using: Entities::LabelBasic + expose :position + end + + class Board < Grape::Entity + expose :id + expose :lists, using: Entities::List do |board| + board.lists.destroyable + end + end + class Compare < Grape::Entity expose :commit, using: Entities::RepoCommit do |compare, options| Commit.decorate(compare.commits, nil).last @@ -495,6 +511,8 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :koding_enabled + expose :koding_url end class Release < Grape::Entity @@ -546,6 +564,10 @@ module API expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -553,6 +575,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -563,8 +586,8 @@ module API expose :key, :value end - class Pipeline < Grape::Entity - expose :id, :status, :ref, :sha, :before_sha, :tag, :yaml_errors + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e9abd53e62b..8ca4cfa68cd 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -6,6 +6,8 @@ module API resource :groups do # Get a groups list # + # Parameters: + # skip_groups (optional) - Array of group ids to exclude from list # Example Request: # GET /groups get do @@ -16,6 +18,7 @@ module API end @groups = @groups.search(params[:search]) if params[:search].present? + @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? @groups = paginate @groups present @groups, with: Entities::Group end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9d71e90075a..d05841def89 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -21,8 +21,11 @@ module API end # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints def find_user_from_warden - warden ? warden.authenticate : nil + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end def find_user_by_private_token @@ -68,6 +71,10 @@ module API @project ||= find_project(params[:id]) end + def available_labels + @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + end + def find_project(id) project = Project.find_with_namespace(id) || Project.find_by(id: id) @@ -115,7 +122,7 @@ module API end def find_project_label(id) - label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') end @@ -194,16 +201,11 @@ module API def validate_label_params(params) errors = {} - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) + params[:labels].to_s.split(',').each do |label_name| + label = available_labels.find_or_initialize_by(title: label_name.strip) + next if label.valid? - if label.invalid? - errors[label.title] = label.errors - end - end + errors[label.title] = label.errors end errors @@ -430,7 +432,7 @@ module API end def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp + Gitlab::Shell.secret_token end def send_git_blob(repository, blob) diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2b723b79504..767f27ef334 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -4,10 +4,9 @@ module API before { authenticate! } resource :keys do - # Get single ssh key by id. Only available to admin users. - # - # Example Request: - # GET /keys/:id + desc 'Get single ssh key by id. Only available to admin users' do + success Entities::SSHKeyWithUser + end get ":id" do authenticated_as_admin! diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c806829d69e..326e1e7ae00 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,37 +3,32 @@ module API class Labels < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end get ':id/labels' do - present user_project.labels, with: Entities::Label, current_user: current_user + present available_labels, with: Entities::Label, current_user: current_user end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = user_project.find_label(attrs[:name]) + label = user_project.find_label(params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + label = user_project.labels.create(declared(params, include_parent_namespaces: false).to_h) if label.valid? present label, with: Entities::Label, current_user: current_user @@ -42,54 +37,44 @@ module API end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + update_params['name'] = update_params.delete('new_name') if update_params.key?('new_name') - if label.update(attrs) + if label.update(update_params) present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) diff --git a/lib/api/license_templates.rb b/lib/api/license_templates.rb deleted file mode 100644 index d0552299ed0..00000000000 --- a/lib/api/license_templates.rb +++ /dev/null @@ -1,58 +0,0 @@ -module API - # License Templates API - class LicenseTemplates < Grape::API - PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze - - # Get the list of the available license templates - # - # Parameters: - # popular - Filter licenses to only the popular ones - # - # Example Request: - # GET /licenses - # GET /licenses?popular=1 - get 'licenses' do - options = { - featured: params[:popular].present? ? true : nil - } - present Licensee::License.all(options), with: Entities::RepoLicense - end - - # Get text for specific license - # - # Parameters: - # key (required) - The key of a license - # project - Copyrighted project name - # fullname - Full name of copyright holder - # - # Example Request: - # GET /licenses/mit - # - get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do - required_attributes! [:key] - - not_found!('License') unless Licensee::License.find(params[:key]) - - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - license = Licensee::License.new(params[:key]) - - license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - - present license, with: Entities::RepoLicense - end - end -end diff --git a/lib/api/members.rb b/lib/api/members.rb index 37f0a6512f4..b80818f0eb6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -5,16 +5,16 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project members viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # query - Query string - # - # Example Request: - # GET /groups/:id/members - # GET /projects/:id/members + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + end get ":id/members" do source = find_source(source_type, params[:id]) @@ -25,15 +25,12 @@ module API present users, with: Entities::Member, source: source end - # Get a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # GET /groups/:id/members/:user_id - # GET /projects/:id/members/:user_id + desc 'Gets a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -43,48 +40,34 @@ module API present member.user, with: Entities::Member, member: member end - # Add a new group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the new member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # POST /groups/:id/members - # POST /projects/:id/members + desc 'Adds a member to a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] - - access_requester = source.requesters.find_by(user_id: params[:user_id]) - if access_requester - # We pass current_user = access_requester so that the requester doesn't - # receive a "access denied" email - ::Members::DestroyService.new(access_requester, access_requester.user).execute - end member = source.members.find_by(user_id: params[:user_id]) - # This is to ensure back-compatibility but 409 behavior should be used - # for both project and group members in 9.0! + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! conflict!('Member already exists') if source_type == 'group' && member unless member - source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - member = source.members.find_by(user_id: params[:user_id]) + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end - if member + if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # Since `source.add_user` doesn't return a member object, we have to - # build a new one and populate its errors in order to render them. - member = source.members.build(attributes_for_keys([:user_id, :access_level, :expires_at])) - member.valid? # populate the errors - # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) @@ -92,21 +75,17 @@ module API end end - # Update a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # PUT /groups/:id/members/:user_id - # PUT /projects/:id/members/:user_id + desc 'Updates a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end put ":id/members/:user_id" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] member = source.members.find_by!(user_id: params[:user_id]) attrs = attributes_for_keys [:access_level, :expires_at] @@ -121,18 +100,12 @@ module API end end - # Remove a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # DELETE /groups/:id/members/:user_id - # DELETE /projects/:id/members/:user_id + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - required_attributes! [:user_id] # This is to ensure back-compatibility but find_by! should be used # in that casse in 9.0! @@ -147,7 +120,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(member, current_user).execute + ::Members::DestroyService.new(source, current_user, declared(params)).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 2b685621da9..bf8504e1101 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -86,14 +86,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute if merge_request.valid? - # Find or create labels and attach to issue - if params[:labels].present? - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors @@ -195,15 +192,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..9b73f6826cf 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -108,8 +108,7 @@ module API finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: @milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 25dc41a3349..cd398274507 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -4,20 +4,18 @@ module API before { authenticate! } resource :namespaces do - # Get a namespaces list - # - # Example Request: - # GET /namespaces + desc 'Get a namespaces list' do + success Entities::Namespace + end + params do + optional :search, type: String, desc: "Search query for namespaces" + end get do - @namespaces = if current_user.admin? - Namespace.all - else - current_user.namespaces - end - @namespaces = @namespaces.search(params[:search]) if params[:search].present? - @namespaces = paginate @namespaces + namespaces = current_user.admin? ? Namespace.all : current_user.namespaces + + namespaces = namespaces.search(params[:search]) if params[:search].present? - present @namespaces, with: Entities::Namespace + present paginate(namespaces), with: Entities::Namespace end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6d99617b56f..da16e24d7ea 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -22,14 +22,25 @@ module API # Example Request: # GET /projects get do - @projects = current_user.authorized_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - if params[:simple] - present @projects, with: Entities::BasicProjectDetails, user: current_user - else - present @projects, with: Entities::ProjectWithAccess, user: current_user - end + projects = current_user.authorized_projects + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user + end + + # Get a list of visible projects for authenticated user + # + # Example Request: + # GET /projects/visible + get '/visible' do + projects = ProjectsFinder.new.execute(current_user) + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user end # Get an owned projects list for authenticated user @@ -37,10 +48,10 @@ module API # Example Request: # GET /projects/owned get '/owned' do - @projects = current_user.owned_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = current_user.owned_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Gets starred project for the authenticated user @@ -48,10 +59,10 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.viewable_starred_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::Project, user: current_user + projects = current_user.viewable_starred_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::Project, user: current_user end # Get all projects for admin user @@ -60,10 +71,10 @@ module API # GET /projects/all get '/all' do authenticated_as_admin! - @projects = Project.all - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = Project.all + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Get a single project @@ -393,23 +404,30 @@ module API # Share project with group # # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group + # id (required) - The ID of a project + # group_id (required) - The ID of a group # group_access (required) - Level of permissions for sharing + # expires_at (optional) - Share expiration date # # Example Request: # POST /projects/:id/share post ":id/share" do authorize! :admin_project, user_project required_attributes! [:group_id, :group_access] + attrs = attributes_for_keys [:group_id, :group_access, :expires_at] + + group = Group.find_by_id(attrs[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(attrs) + if link.save present link, with: Entities::ProjectGroupLink else diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 22b8f90dc5c..794e34874f4 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -7,38 +7,36 @@ module API end resource :hooks do - # Get the list of system hooks - # - # Example Request: - # GET /hooks + desc 'Get the list of system hooks' do + success Entities::Hook + end get do - @hooks = SystemHook.all - present @hooks, with: Entities::Hook + hooks = SystemHook.all + present hooks, with: Entities::Hook end - # Create new system hook - # - # Parameters: - # url (required) - url for system hook - # Example Request - # POST /hooks + desc 'Create a new system hook' do + success Entities::Hook + end + params do + requires :url, type: String, desc: 'The URL for the system hook' + end post do - attrs = attributes_for_keys [:url] - required_attributes! [:url] - @hook = SystemHook.new attrs - if @hook.save - present @hook, with: Entities::Hook + hook = SystemHook.new declared(params).to_h + + if hook.save + present hook, with: Entities::Hook else not_found! end end - # Test a hook - # - # Example Request - # GET /hooks/:id + desc 'Test a hook' + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end get ":id" do - @hook = SystemHook.find(params[:id]) + hook = SystemHook.find(params[:id]) data = { event_name: "project_create", name: "Ruby", @@ -47,23 +45,21 @@ module API owner_name: "Someone", owner_email: "example@gitlabhq.com" } - @hook.execute(data, 'system_hooks') + hook.execute(data, 'system_hooks') data end - # Delete a hook. This is an idempotent function. - # - # Parameters: - # id (required) - ID of the hook - # Example Request: - # DELETE /hooks/:id + desc 'Delete a hook' do + success Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end delete ":id" do - begin - @hook = SystemHook.find(params[:id]) - @hook.destroy - rescue - # SystemHook raises an Error if no hook with id found - end + hook = SystemHook.find_by(id: params[:id]) + not_found!('System hook') unless hook + + present hook.destroy, with: Entities::Hook end end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index b9e718147e1..8a53d9c0095 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,39 +1,115 @@ module API class Templates < Grape::API GLOBAL_TEMPLATE_TYPES = { - gitignores: Gitlab::Template::GitignoreTemplate, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + } }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + DEPRECATION_MESSAGE = ' This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + def render_response(template_type, template) not_found!(template_type.to_s.singularize) unless template present template, with: Entities::Template end end - GLOBAL_TEMPLATE_TYPES.each do |template_type, klass| - # Get the list of the available template - # - # Example Request: - # GET /gitignores - # GET /gitlab_ci_ymls - get template_type.to_s do - present klass.all, with: Entities::TemplatesList - end - - # Get the text for a specific template present in local filesystem - # - # Parameters: - # name (required) - The name of a template - # - # Example Request: - # GET /gitignores/Elixir - # GET /gitlab_ci_ymls/Ruby - get "#{template_type}/:name" do - required_attributes! [:name] - new_template = klass.find(params[:name]) - render_response(template_type, new_template) + { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| + desc 'Get the list of the available license template' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + end + get route do + options = { + featured: declared(params).popular.present? ? true : nil + } + present Licensee::License.all(options), with: Entities::RepoLicense + end + end + + { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific license' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route, requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: Entities::RepoLicense + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| + desc 'Get the list of the available template' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::TemplatesList + end + get route do + present klass.all, with: Entities::TemplatesList + end + end + + { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific template present in local filesystem' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route do + new_template = klass.find(declared(params).name) + + render_response(template_type, new_template) + end end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 19df13d8aac..832b04a3bb1 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -8,18 +8,19 @@ module API 'issues' => ->(id) { find_project_issue(id) } } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_id".to_sym - # Create a todo on an issuable - # - # Parameters: - # id (required) - The ID of a project - # issuable_id (required) - The ID of an issuable - # Example Request: - # POST /projects/:id/issues/:issuable_id/todo - # POST /projects/:id/merge_requests/:issuable_id/todo + desc 'Create a todo on an issuable' do + success Entities::Todo + end + params do + requires type_id_str, type: Integer, desc: 'The ID of an issuable' + end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) todo = TodoService.new.mark_todo(issuable, current_user).first @@ -40,25 +41,21 @@ module API end end - # Get a todo list - # - # Example Request: - # GET /todos - # + desc 'Get a todo list' do + success Entities::Todo + end get do todos = find_todos present paginate(todos), with: Entities::Todo, current_user: current_user end - # Mark a todo as done - # - # Parameters: - # id: (required) - The ID of the todo being marked as done - # - # Example Request: - # DELETE /todos/:id - # + desc 'Mark a todo as done' do + success Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end delete ':id' do todo = current_user.todos.find(params[:id]) TodoService.new.mark_todos_as_done([todo], current_user) @@ -66,11 +63,7 @@ module API present todo.reload, with: Entities::Todo, current_user: current_user end - # Mark all todos as done - # - # Example Request: - # DELETE /todos - # + desc 'Mark all todos as done' delete do todos = find_todos TodoService.new.mark_todos_as_done(todos, current_user) diff --git a/lib/api/users.rb b/lib/api/users.rb index 4924182770b..6ab9786a97a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -338,6 +338,26 @@ module API user.activate end end + + desc 'Get contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + requires :id, type: String, desc: 'The user ID' + end + get ':id/events' do + user = User.find_by(id: declared(params).id) + not_found!('User') unless user + + events = user.recent_events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + page(params[:page]) + + present paginate(events), with: Entities::Event + end end resource :user do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f6495071a11..b9fb3c21dbb 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -4,27 +4,29 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do - # Get project variables - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/variables + desc 'Get project variables' do + success Entities::Variable + end + params do + optional :page, type: Integer, desc: 'The page number for pagination' + optional :per_page, type: Integer, desc: 'The value of items per page to show' + end get ':id/variables' do variables = user_project.variables present paginate(variables), with: Entities::Variable end - # Get specific variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The `key` of variable - # Example Request: - # GET /projects/:id/variables/:key + desc 'Get a specific variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end get ':id/variables/:key' do key = params[:key] variable = user_project.variables.find_by(key: key.to_s) @@ -34,18 +36,15 @@ module API present variable, with: Entities::Variable end - # Create a new variable in project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The key of variable - # value (required) - The value of variable - # Example Request: - # POST /projects/:id/variables + desc 'Create a new variable in a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end post ':id/variables' do - required_attributes! [:key, :value] - - variable = user_project.variables.create(key: params[:key], value: params[:value]) + variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) if variable.valid? present variable, with: Entities::Variable @@ -54,41 +53,37 @@ module API end end - # Update existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (optional) - The `key` of variable - # value (optional) - New value for `value` field of variable - # Example Request: - # PUT /projects/:id/variables/:key + desc 'Update an existing variable from a project' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end put ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - attrs = attributes_for_keys [:value] - if variable.update(attrs) + if variable.update(value: params[:value]) present variable, with: Entities::Variable else render_validation_error!(variable) end end - # Delete existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The ID of a variable - # Example Request: - # DELETE /projects/:id/variables/:key + desc 'Delete an existing variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end delete ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - variable.destroy - present variable, with: Entities::Variable + present variable.destroy, with: Entities::Variable end end end diff --git a/lib/api/version.rb b/lib/api/version.rb new file mode 100644 index 00000000000..9ba576bd828 --- /dev/null +++ b/lib/api/version.rb @@ -0,0 +1,12 @@ +module API + class Version < Grape::API + before { authenticate! } + + desc 'Get the version information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 8.13.' + end + get '/version' do + { version: Gitlab::VERSION, revision: Gitlab::REVISION } + end + end +end diff --git a/lib/banzai.rb b/lib/banzai.rb index 9ebe379f454..35ca234c1ba 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.render_field(object, field) + Renderer.render_field(object, field) + end + def self.cache_collection_render(texts_and_contexts) Renderer.cache_collection_render(texts_and_contexts) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index affe34394c2..cb213a76a05 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -208,8 +208,12 @@ module Banzai @references_per_project ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = Regexp.union(object_class.reference_pattern, - object_class.link_reference_pattern) + regex = + if uses_reference_pattern? + Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) + else + object_class.link_reference_pattern + end nodes.each do |node| node.to_html.scan(regex) do @@ -295,6 +299,14 @@ module Banzai value end end + + # There might be special cases like filters + # that should ignore reference pattern + # eg: IssueReferenceFilter when using a external issues tracker + # In those cases this method should be overridden on the filter subclass + def uses_reference_pattern? + true + end end end end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 2492b5213ac..a8c1ca0c60a 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,6 +1,6 @@ module Banzai module Filter - # HTML filter that replaces :emoji: with images. + # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter # @@ -13,16 +13,17 @@ module Banzai def call search_text_nodes(doc).each do |node| content = node.to_html - next unless content.include?(':') next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - html = emoji_image_filter(content) + next unless content.include?(':') || node.text.match(emoji_unicode_pattern) + + html = emoji_name_image_filter(content) + html = emoji_unicode_image_filter(html) next if html == content node.replace(html) end - doc end @@ -31,18 +32,38 @@ module Banzai # text - String text to replace :emoji: in. # # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) + def emoji_name_image_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />" + emoji_image_tag(name, emoji_url(name)) end end + # Replace unicode emoji with corresponding images if they exist. + # + # text - String text to replace unicode emoji in. + # + # Returns a String with unicode emoji replaced with images. + def emoji_unicode_image_filter(text) + text.gsub(emoji_unicode_pattern) do |moji| + emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + end + end + + def emoji_image_tag(emoji_name, emoji_url) + "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" + end + # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end + # Build a regexp that matches all valid unicode emojis names. + def self.emoji_unicode_pattern + @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + end + private def emoji_url(name) @@ -60,6 +81,18 @@ module Banzai end end + def emoji_unicode_url(moji) + emoji_unicode_path = emoji_unicode_filename(moji) + + if context[:asset_host] + url_to_image(emoji_unicode_path) + elsif context[:asset_root] + File.join(context[:asset_root], url_to_image(emoji_unicode_path)) + else + url_to_image(emoji_unicode_path) + end + end + def url_to_image(image) ActionController::Base.helpers.url_to_image(image) end @@ -71,6 +104,14 @@ module Banzai def emoji_filename(name) "#{Gitlab::Emoji.emoji_filename(name)}.png" end + + def emoji_unicode_pattern + self.class.emoji_unicode_pattern + end + + def emoji_unicode_filename(name) + "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index eaa702952cc..0d20be557a0 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -8,7 +8,7 @@ module Banzai # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text) do |match, issue| + # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -17,8 +17,8 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(ExternalIssue.reference_pattern) do |match| + def self.references_in(text, pattern) + text.gsub(pattern) do |match| yield match, $~[:issue] end end @@ -27,7 +27,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = ExternalIssue.reference_pattern + ref_pattern = issue_reference_pattern ref_start_pattern = /\A#{ref_pattern}\z/ each_node do |node| @@ -60,7 +60,7 @@ module Banzai def issue_link_filter(text, link_text: nil) project = context[:project] - self.class.references_in(text) do |match, id| + self.class.references_in(text, issue_reference_pattern) do |match, id| ExternalIssue.new(id, project) url = url_for_issue(id, project, only_path: context[:only_path]) @@ -82,18 +82,21 @@ module Banzai end def default_issues_tracker? - if RequestStore.active? - default_issues_tracker_cache[project.id] ||= - project.default_issues_tracker? - else - project.default_issues_tracker? - end + external_issues_cached(:default_issues_tracker?) + end + + def issue_reference_pattern + external_issues_cached(:issue_reference_pattern) end private - def default_issues_tracker_cache - RequestStore[:banzai_default_issues_tracker_cache] ||= {} + def external_issues_cached(attribute) + return project.public_send(attribute) unless RequestStore.active? + + cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 0a29c547a4d..2f19b59e725 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -3,10 +3,17 @@ module Banzai # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - # Skip non-HTTP(S) links and internal links - doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| - node.set_attribute('rel', 'nofollow noreferrer') - node.set_attribute('target', '_blank') + links.each do |node| + href = href_to_lowercase_scheme(node["href"].to_s) + + unless node["href"].to_s == href + node.set_attribute('href', href) + end + + if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') + end end doc @@ -14,6 +21,25 @@ module Banzai private + def links + query = 'descendant-or-self::a[@href and not(@href = "")]' + doc.xpath(query) + end + + def href_to_lowercase_scheme(href) + scheme_match = href.match(/\A(\w+):\/\//) + + if scheme_match + scheme_match.to_s.downcase + scheme_match.post_match + else + href + end + end + + def external_url?(url) + !url.start_with?(internal_url) + end + def internal_url @internal_url ||= Gitlab.config.gitlab.url end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb new file mode 100644 index 00000000000..f3bd587c28b --- /dev/null +++ b/lib/banzai/filter/html_entity_filter.rb @@ -0,0 +1,12 @@ +require 'erb' + +module Banzai + module Filter + # Text filter that escapes these HTML entities: & " < > + class HtmlEntityFilter < HTML::Pipeline::TextFilter + def call + ERB::Util.html_escape_once(text) + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 54c5f9a71a4..4d1bc687696 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -4,6 +4,10 @@ module Banzai # issues that do not exist are ignored. # # This filter supports cross-project references. + # + # When external issues tracker like Jira is activated we should not + # use issue reference pattern, but we should still be able + # to reference issues from other GitLab projects. class IssueReferenceFilter < AbstractReferenceFilter self.reference_type = :issue @@ -11,6 +15,10 @@ module Banzai Issue end + def uses_reference_pattern? + context[:project].default_issues_tracker? + end + def find_object(project, iid) issues_per_project[project][iid] end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8f262ef3d8d..c24831e68ee 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -9,7 +9,7 @@ module Banzai end def find_object(project, id) - project.labels.find(id) + find_labels(project).find(id) end def self.references_in(text, pattern = Label.reference_pattern) @@ -35,7 +35,11 @@ module Banzai return unless project label_params = label_params(label_id, label_name) - project.labels.find_by(label_params) + find_labels(project).find_by(label_params) + end + + def find_labels(project) + LabelsFinder.new(nil, project_id: project.id).execute(authorized_only: false) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -60,13 +64,50 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - LabelsHelper.render_colored_label(object) + if same_group?(object) && namespace_match?(matches) + render_same_project_label(object) + elsif same_project?(object) + render_same_project_label(object) else - LabelsHelper.render_colored_cross_project_label(object) + render_cross_project_label(object, matches) end end + def same_group?(object) + object.is_a?(GroupLabel) && object.group == project.group + end + + def namespace_match?(matches) + matches[:project].blank? || matches[:project] == project.path_with_namespace + end + + def same_project?(object) + object.is_a?(ProjectLabel) && object.project == project + end + + def user + context[:current_user] || context[:author] + end + + def project + context[:project] + end + + def render_same_project_label(object) + LabelsHelper.render_colored_label(object) + end + + def render_cross_project_label(object, matches) + source_project = + if matches[:project] + Project.find_with_namespace(matches[:project]) + else + object.project + end + + LabelsHelper.render_colored_cross_project_label(object, source_project) + end + def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 2470362e019..af1e575fc89 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,7 +25,7 @@ module Banzai return if customized?(whitelist[:transformers]) # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) + whitelist[:attributes]['pre'] = %w(class v-pre) whitelist[:attributes]['span'] = %w(class) # Allow table alignment diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb new file mode 100644 index 00000000000..c2976aeb7c6 --- /dev/null +++ b/lib/banzai/filter/set_direction_filter.rb @@ -0,0 +1,15 @@ +module Banzai + module Filter + # HTML filter that sets dir="auto" for RTL languages support + class SetDirectionFilter < HTML::Pipeline::Filter + def call + # select these elements just on top level of the document + doc.xpath('p|h1|h2|h3|h4|h5|h6|ol|ul[not(@class="section-nav")]|blockquote|table').each do |el| + el['dir'] = 'auto' + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fcdb496aed2..026b81ac175 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -30,7 +30,7 @@ module Banzai # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..9fa5f589f3e 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,27 +2,7 @@ require 'task_list/filter' module Banzai module Filter - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) - if new_class_names.include?('task-list') - # Don't add class to all lists - return - elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') - end - - add_css_class_without_fix(node, *new_class_names) - end - - alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index e1ca7f4d24b..c6302b586d3 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -106,13 +106,17 @@ module Banzai project = context[:project] author = context[:author] - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) + if author && !project.team.member?(author) + link_text + else + url = urls.namespace_project_url(project.namespace, project, + only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + data = data_attribute(project: project.id, author: author.try(:id)) + text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, text, 'All Project and Group Members') + end end def link_to_namespace(namespace, link_text: nil) diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb index bab6a9934d1..2b7c10f1a0e 100644 --- a/lib/banzai/note_renderer.rb +++ b/lib/banzai/note_renderer.rb @@ -3,7 +3,7 @@ module Banzai # Renders a collection of Note instances. # # notes - The notes to render. - # project - The project to use for rendering/redacting. + # project - The project to use for redacting. # user - The user viewing the notes. # path - The request path. # wiki - The project's wiki. @@ -13,8 +13,7 @@ module Banzai user, requested_path: path, project_wiki: wiki, - ref: git_ref, - pipeline: :note) + ref: git_ref) renderer.render(notes, :note) end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9aef807c152..9f8eb0931b8 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -1,28 +1,32 @@ module Banzai - # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # Class for rendering multiple objects (e.g. Note instances) in a single pass, + # using +render_field+ to benefit from caching in the database. Rendering and + # redaction are both performed. # - # Rendered Markdown is stored in an attribute in every object based on the - # name of the attribute containing the Markdown. For example, when the - # attribute `note` is rendered the HTML is stored in `note_html`. + # The unredacted HTML is generated according to the usual +render_field+ + # policy, so specify the pipeline and any other context options on the model. + # + # The *redacted* (i.e., suitable for use) HTML is placed in an attribute + # named "redacted_<foo>", where <foo> is the name of the cache field for the + # chosen attribute. + # + # As an example, rendering the attribute `note` would place the unredacted + # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer attr_reader :project, :user - # Make sure to set the appropriate pipeline in the `raw_context` attribute - # (e.g. `:note` for Note instances). - # - # project - A Project to use for rendering and redacting Markdown. + # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use in the rendering - # pipeline. - def initialize(project, user = nil, raw_context = {}) + # context - A Hash containing extra attributes to use during redaction + def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @raw_context = raw_context + @redaction_context = redaction_context end # Renders and redacts an Array of objects. # - # objects - The objects to render + # objects - The objects to render. # attribute - The attribute containing the raw Markdown to render. # # Returns the same input objects. @@ -32,7 +36,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) object.user_visible_reference_count = redacted_data[:visible_reference_count] end end @@ -53,12 +57,8 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.merge(cache_key: [object, attribute]) - - if object.respond_to?(:author) - context[:author] = object.author - end - + context = base_context.dup + context = context.merge(object.banzai_render_context(attribute)) context end @@ -66,21 +66,16 @@ module Banzai # # Returns an Array of `Nokogiri::HTML::Document`. def render_attributes(objects, attribute) - strings_and_contexts = objects.map do |object| + objects.map do |object| + string = Banzai.render_field(object, attribute) context = context_for(object, attribute) - string = object.__send__(attribute) - - { text: string, context: context } - end - - Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| - Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + Banzai::Pipeline[:relative_link].to_document(string, context) end end def base_context - @base_context ||= @raw_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge(current_user: user, project: project) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8d94b199c66..5da2d0b008c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -25,7 +25,9 @@ module Banzai Filter::MilestoneReferenceFilter, Filter::TaskListFilter, - Filter::InlineDiffFilter + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index ba2555df98d..1929099931b 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,6 +3,7 @@ module Banzai class SingleLinePipeline < GfmPipeline def self.filters @filters ||= FilterArray[ + Filter::HtmlEntityFilter, Filter::SanitizationFilter, Filter::EmojiFilter, diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index a4ae27eefd8..ce048a36fa0 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,6 +1,6 @@ module Banzai module Renderer - extend self + module_function # Convert a Markdown String into an HTML-safe String of HTML # @@ -31,6 +31,34 @@ module Banzai end end + # Convert a Markdown-containing field on an object into an HTML-safe String + # of HTML. This method is analogous to calling render(object.field), but it + # can cache the rendered HTML in the object, rather than Redis. + # + # The context to use is learned from the passed-in object by calling + # #banzai_render_context(field), and cannot be changed. Use #render, passing + # it the field text, if a custom rendering is needed. The generated context + # is returned along with the HTML. + def render_field(object, field) + html_field = object.markdown_cache_field_for(field) + + html = object.__send__(html_field) + return html if html.present? + + html = cacheless_render_field(object, field) + object.update_column(html_field, html) unless object.new_record? || object.destroyed? + + html + end + + # Same as +render_field+, but without consulting or updating the cache field + def cacheless_render_field(object, field) + text = object.__send__(field) + context = object.banzai_render_context(field) + + cacheless_render(text, context) + end + # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # @@ -113,8 +141,6 @@ module Banzai end.html_safe end - private - def cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 59f85416ee5..ed87a2603e8 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,10 +12,9 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact(save: false) - update_runner_info required_attributes! [:token] not_found! unless current_runner.active? + update_runner_info build = Ci::RegisterBuildService.new.execute(current_runner) @@ -41,10 +40,11 @@ module Ci # PUT /builds/:id put ":id" do authenticate_runner! - update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) forbidden!('Build has been erased!') if build.erased? + update_runner_info + build.update_attributes(trace: params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 23353c62885..e608f5f6cad 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 40 * 60 + UPDATE_RUNNER_EVERY = 10 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -30,14 +30,22 @@ module Ci token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact(save: true) - # Use a random threshold to prevent beating DB updates - # it generates a distribution between: [40m, 80m] + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.contacted_at = Time.now - current_runner.save if current_runner.changed? && save - end + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age end def build_not_found! @@ -57,11 +65,6 @@ module Ci attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) end - def update_runner_info - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - def max_artifacts_size current_application_settings.max_artifacts_size.megabytes.to_i end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 0369e80312a..3e33c9399e2 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,7 +4,7 @@ module Ci include Gitlab::Ci::Config::Node::LegacyValidationHelpers - attr_reader :path, :cache, :stages + attr_reader :path, :cache, :stages, :jobs def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -109,6 +109,7 @@ module Ci validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_environment!(name, job) end end @@ -150,6 +151,35 @@ module Ci end end + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb new file mode 100644 index 00000000000..ca39b1961ae --- /dev/null +++ b/lib/constraints/group_url_constrainer.rb @@ -0,0 +1,7 @@ +require 'constraints/namespace_url_constrainer' + +class GroupUrlConstrainer < NamespaceUrlConstrainer + def find_resource(id) + Group.find_by_path(id) + end +end diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb new file mode 100644 index 00000000000..91b70143f11 --- /dev/null +++ b/lib/constraints/namespace_url_constrainer.rb @@ -0,0 +1,24 @@ +class NamespaceUrlConstrainer + def matches?(request) + id = request.path + id = id.sub(/\A#{relative_url_root}/, '') if relative_url_root + id = id.sub(/\A\/+/, '').split('/').first + id = id.sub(/.atom\z/, '') if id + + if id =~ Gitlab::Regex.namespace_regex + find_resource(id) + end + end + + def find_resource(id) + Namespace.find_by_path(id) + end + + private + + def relative_url_root + if defined?(Gitlab::Application.config.relative_url_root) + Gitlab::Application.config.relative_url_root + end + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb new file mode 100644 index 00000000000..504a0f5d93e --- /dev/null +++ b/lib/constraints/user_url_constrainer.rb @@ -0,0 +1,7 @@ +require 'constraints/namespace_url_constrainer' + +class UserUrlConstrainer < NamespaceUrlConstrainer + def find_resource(id) + User.find_by('lower(username) = ?', id.downcase) + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 668d2fa41b3..21f6a9a762b 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -2,8 +2,8 @@ class EventFilter attr_accessor :params class << self - def default_filter - %w{ push issues merge_requests team} + def all + 'all' end def push @@ -35,18 +35,28 @@ class EventFilter return events unless params.present? filter = params.dup - actions = [] - actions << Event::PUSHED if filter.include? 'push' - actions << Event::MERGED if filter.include? 'merged' - if filter.include? 'team' - actions << Event::JOINED - actions << Event::LEFT + case filter + when EventFilter.push + actions = [Event::PUSHED] + when EventFilter.merged + actions = [Event::MERGED] + when EventFilter.comments + actions = [Event::COMMENTED] + when EventFilter.team + actions = [Event::JOINED, Event::LEFT, Event::EXPIRED] + when EventFilter.all + actions = [ + Event::PUSHED, + Event::MERGED, + Event::COMMENTED, + Event::JOINED, + Event::LEFT, + Event::EXPIRED + ] end - actions << Event::COMMENTED if filter.include? 'comments' - events.where(action: actions) end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a4558d157c0..9b74364849e 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -52,8 +52,7 @@ module ExtractsPath # Append a trailing slash if we only get a ref and no file path id += '/' unless id.ends_with?('/') - valid_refs = @project.repository.ref_names - valid_refs.select! { |v| id.start_with?("#{v}/") } + valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } if valid_refs.length == 0 # No exact ref match, so just try our best @@ -74,6 +73,19 @@ module ExtractsPath pair end + # If we have an ID of 'foo.atom', and the controller provides Atom and HTML + # formats, then we have to check if the request was for the Atom version of + # the ID without the '.atom' suffix, or the HTML version of the ID including + # the suffix. We only check this if the version including the suffix doesn't + # match, so it is possible to create a branch which has an unroutable Atom + # feed. + def extract_ref_without_atom(id) + id_without_atom = id.sub(/\.atom$/, '') + valid_refs = ref_names.select { |v| "#{id_without_atom}/".start_with?("#{v}/") } + + valid_refs.max_by(&:length) + end + # Assigns common instance variables for views working with Git tree-ish objects # # Assignments are: @@ -86,6 +98,10 @@ module ExtractsPath # If the :id parameter appears to be requesting a specific response format, # that will be handled as well. # + # If there is no path and the ref doesn't exist in the repo, try to resolve + # the ref without an '.atom' suffix. If _that_ ref is found, set the request's + # format to Atom manually. + # # Automatically renders `not_found!` if a valid tree path could not be # resolved (e.g., when a user inserts an invalid path or ref). def assign_ref_vars @@ -97,10 +113,18 @@ module ExtractsPath @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository - if @options[:extended_sha1].blank? - @commit = @repo.commit(@ref) - else + + if @options[:extended_sha1].present? @commit = @repo.commit(@options[:extended_sha1]) + else + @commit = @repo.commit(@ref) + + if @path.empty? && !@commit && @id.ends_with?('.atom') + @id = @ref = extract_ref_without_atom(@id) + @commit = @repo.commit(@ref) + + request.format = :atom if @commit + end end raise InvalidPathError unless @commit @@ -125,4 +149,10 @@ module ExtractsPath id += "/" + params[:path] unless params[:path].blank? id end + + def ref_names + return [] unless @project + + @ref_names ||= @project.repository.ref_names + end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index a533bac2692..9b484a2ecfd 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -53,6 +53,10 @@ module Gitlab } end + def sym_options_with_owner + sym_options.merge(owner: OWNER) + end + def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 79eac66b364..9cec71a3222 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -17,6 +17,18 @@ module Gitlab end class << self + def secret_token + @secret_token ||= begin + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + end + + def ensure_secret_token! + return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')) + + generate_and_link_secret_token + end + def version_required @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip @@ -25,6 +37,25 @@ module Gitlab def strip_key(key) key.split(/ /)[0, 2].join(' ') end + + private + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + shell_path = Gitlab.config.gitlab_shell.path + + unless File.size?(secret_file) + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + @secret_token = SecureRandom.hex(16) + File.write(secret_file, @secret_token) + end + + link_path = File.join(shell_path, '.gitlab_shell_secret') + if File.exist?(shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end + end end # Init new repository @@ -201,21 +232,6 @@ module Gitlab File.exist?(full_path(storage, dir_name)) end - # Create (if necessary) and link the secret token file - def generate_and_link_secret_token - secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.size?(secret_file) - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) - end - - link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') - if File.exist?(gitlab_shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) - end - end - protected def gitlab_shell_path diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb index d388ab6b879..9a95ef43628 100644 --- a/lib/gitlab/ci/config/node/environment.rb +++ b/lib/gitlab/ci/config/node/environment.rb @@ -8,7 +8,7 @@ module Gitlab class Environment < Entry include Validatable - ALLOWED_KEYS = %i[name url] + ALLOWED_KEYS = %i[name url action on_stop] validations do validate do @@ -35,6 +35,12 @@ module Gitlab length: { maximum: 255 }, addressable_url: true, allow_nil: true + + validates :action, + inclusion: { in: %w[start stop], message: 'should be start or stop' }, + allow_nil: true + + validates :on_stop, type: String, allow_nil: true end end @@ -54,9 +60,17 @@ module Gitlab value[:url] end + def action + value[:action] || 'start' + end + + def on_stop + value[:on_stop] + end + def value case @config - when String then { name: @config } + when String then { name: @config, action: 'start' } when Hash then @config else {} end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb new file mode 100644 index 00000000000..37e51536e8f --- /dev/null +++ b/lib/gitlab/ci/trace_reader.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class TraceReader + BUFFER_SIZE = 4096 + + attr_accessor :path, :buffer_size + + def initialize(new_path, buffer_size: BUFFER_SIZE) + self.path = new_path + self.buffer_size = Integer(buffer_size) + end + + def read(last_lines: nil) + if last_lines + read_last_lines(last_lines) + else + File.read(path) + end + end + + def read_last_lines(max_lines) + File.open(path) do |file| + chunks = [] + pos = lines = 0 + max = file.size + + # We want an extra line to make sure fist line has full contents + while lines <= max_lines && pos < max + pos += buffer_size + + buf = if pos <= max + file.seek(-pos, IO::SEEK_END) + file.read(buffer_size) + else # Reached the head, read only left + file.seek(0) + file.read(buffer_size - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(max_lines).join + end + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index dff9e29c6a5..c843315782d 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,7 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers include IconsHelper - class MissingResolution < StandardError + class MissingResolution < ResolutionError end CONTEXT_LINES = 3 @@ -21,12 +21,34 @@ module Gitlab @match_line_headers = {} end + def content + merge_file_result[:data] + end + + def our_blob + @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) + end + + def type + lines unless @type + + @type.inquiry + end + # Array of Gitlab::Diff::Line objects def lines - @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data], + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Conflict::Parser.new.parse(content, our_path: our_path, their_path: their_path, parent_file: self) + rescue Gitlab::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end end def resolve_lines(resolution) @@ -53,6 +75,14 @@ module Gitlab end.compact end + def resolve_content(resolution) + if resolution == content + raise MissingResolution, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + def highlight_lines! their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") @@ -170,21 +200,39 @@ module Gitlab match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" end - def as_json(opts = nil) - { + def as_json(opts = {}) + json_hash = { old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), blob_path: namespace_project_blob_path(merge_request.project.namespace, merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)), - sections: sections + ::File.join(merge_request.diff_refs.head_sha, our_path)) } + + json_hash.tap do |json_hash| + if opts[:full_content] + json_hash[:content] = content + json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode) + else + json_hash[:sections] = sections if type.text? + json_hash[:type] = type + json_hash[:content_path] = content_path + end + end + end + + def content_path + conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, + merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. def inspect - instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode].map do |instance_variable| + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| value = instance_variable_get("@#{instance_variable}") "#{instance_variable}=\"#{value}\"" diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index bbd0427a2c8..fa5bd4649d4 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -30,6 +30,10 @@ module Gitlab end end + def file_for_path(old_path, new_path) + files.find { |file| file.their_path == old_path && file.our_path == new_path } + end + def as_json(opts = nil) { target_branch: merge_request.target_branch, diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 98e842cded3..ddd657903fb 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -1,19 +1,24 @@ module Gitlab module Conflict class Parser - class ParserError < StandardError + class UnresolvableError < StandardError end - class UnexpectedDelimiter < ParserError + class UnmergeableFile < UnresolvableError end - class MissingEndDelimiter < ParserError + class UnsupportedEncoding < UnresolvableError + end + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + class ParserError < StandardError end - class UnmergeableFile < ParserError + class UnexpectedDelimiter < ParserError end - class UnsupportedEncoding < ParserError + class MissingEndDelimiter < ParserError end def parse(text, our_path:, their_path:, parent_file: nil) diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb new file mode 100644 index 00000000000..a0f2006bc24 --- /dev/null +++ b/lib/gitlab/conflict/resolution_error.rb @@ -0,0 +1,6 @@ +module Gitlab + module Conflict + class ResolutionError < StandardError + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e47df508ca2..ce85e5e0123 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -125,6 +125,10 @@ module Gitlab repository.blob_at(commit.id, file_path) end + + def cache_key + "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 36348b33943..dc4d47c878b 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -35,16 +35,16 @@ module Gitlab # for the highlighted ones, so we just skip their execution. # If the highlighted diff files lines are not cached we calculate and cache them. # - # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of + # The content of the cache is a Hash where the key identifies the file and the values are Arrays of # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - file_path = diff_file.file_path + item_key = diff_file.cache_key - if highlight_cache[file_path] - highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path]) + if highlight_cache[item_key] + highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) else - highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 06dae31cc27..447c7a6a6b9 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -46,7 +46,9 @@ module Gitlab noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code + line_code: sent_notification.line_code, + position: sent_notification.position, + type: sent_notification.note_type ).execute end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index b63213ae208..bbbca8acc40 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -10,12 +10,20 @@ module Gitlab Gemojione.index.instance_variable_get(:@emoji_by_moji) end + def emojis_unicodes + emojis_by_moji.keys + end + def emojis_names - emojis.keys.sort + emojis.keys end def emoji_filename(name) emojis[name]["unicode"] end + + def emoji_unicode_filename(moji) + emojis_by_moji[moji]["unicode"] + end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 501d5a95547..65ee85ca5a9 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -74,8 +74,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, title: name, color: color) + params = { title: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end def user_info(person_id) @@ -122,25 +122,21 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed' + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_id: assignee_id, + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue.add_labels_by_names(labels) - if issue.iid != bug['ixBug'] - issue.update_attribute(:iid, bug['ixBug']) - end + issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) - - issue.update_attribute(:created_at, date) - - last_update = DateTime.parse(bug['dtLastUpdated']) - issue.update_attribute(:updated_at, last_update) end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 78d7a4f27cf..a7c596dced0 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -58,7 +58,7 @@ module Gitlab referable = find_referable(reference) return reference unless referable - cross_reference = referable.to_reference(target_project) + cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference new_text = before + cross_reference + after @@ -72,6 +72,14 @@ module Gitlab extractor.all.first end + def build_cross_reference(referable, target_project) + if referable.respond_to?(:project) + referable.to_reference(target_project) + else + referable.to_reference(@source_project, target_project) + end + end + def substitution_valid?(substituted) @original_html == markdown(substituted) end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index e33ac61f5ae..7f424b74efb 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -102,9 +102,19 @@ module Gitlab def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = api.send(method, *args, &block) - yield data + data = api.send(method, *args) + return data unless data.is_a?(Array) + if block_given? + yield data + each_response_page(&block) + else + each_response_page { |page| data.concat(page) } + data + end + end + + def each_response_page last_response = api.last_response while last_response.rels[:next] diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b8321244473..4b70f33a851 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -170,10 +170,9 @@ module Gitlab end def import_wiki - unless project.wiki_enabled? + unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 2cad7fca88e..942dfb3312b 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -14,9 +14,13 @@ module Gitlab end def create! - project.labels.find_or_create_by!(title: title) do |label| - label.color = color - end + params = attributes.except(:project) + service = ::Labels::FindOrCreateService.new(project.owner, project, params) + label = service.execute + + raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + + label end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 605abfabdab..a2410068845 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,7 +1,7 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data def initialize(repo, name, namespace, current_user, session_data) @repo = repo @@ -12,24 +12,37 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, - name: @name, - path: @name, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, + visibility_level: visibility_level, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: import_url, + skip_wiki: skip_wiki ).execute + end + + private - # If repo has wiki we'll import it later - if repo.has_wiki? && project - project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) - end + def import_url + repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + end - project + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 62da327931f..6a68e786b4f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -92,19 +92,17 @@ module Gitlab end issue = Issue.create!( - project_id: project.id, - title: raw_issue["title"], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue["state"] == "closed" ? "closed" : "opened" + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_id: assignee_id, + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue.add_labels_by_names(labels) - - if issue.iid != raw_issue["id"] - issue.update_attribute(:iid, raw_issue["id"]) - end + issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) end @@ -236,8 +234,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, name: name, color: color) + params = { name: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end def format_content(raw_content) diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3e5d728f3bc..f8809db21aa 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,19 +5,61 @@ module Gitlab def identify(identifier, project, newrev) if identifier.blank? # Local push from gitlab - email = project.commit(newrev).author_email rescue nil - User.find_by(email: email) if email - + identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http - user_id = identifier.gsub("user-", "") - User.find_by(id: user_id) - + identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh - key_id = identifier.gsub("key-", "") - Key.find_by(id: key_id).try(:user) + identify_using_ssh_key(identifier) + end + end + + # Tries to identify a user based on a commit SHA. + def identify_using_commit(project, ref) + commit = project.commit(ref) + + return if !commit || !commit.author_email + + email = commit.author_email + + identify_with_cache(:email, email) do + User.find_by(email: email) end end + + # Tries to identify a user based on a user identifier (e.g. "user-123"). + def identify_using_user(identifier) + user_id = identifier.gsub("user-", "") + + identify_with_cache(:user, user_id) do + User.find_by(id: user_id) + end + end + + # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). + def identify_using_ssh_key(identifier) + key_id = identifier.gsub("key-", "") + + identify_with_cache(:ssh_key, key_id) do + User.find_by_ssh_key_id(key_id) + end + end + + def identify_with_cache(category, key) + if identification_cache[category].key?(key) + identification_cache[category][key] + else + identification_cache[category][key] = yield + end + end + + def identification_cache + @identification_cache ||= { + email: {}, + user: {}, + ssh_key: {} + } + end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 181e288a014..eb667a85b78 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.4' + VERSION = '0.1.5' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb new file mode 100644 index 00000000000..f755a404693 --- /dev/null +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -0,0 +1,13 @@ +module Gitlab + module ImportExport + class AttributeCleaner + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + + def self.clean!(relation_hash:) + relation_hash.reject! do |key, _value| + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + end + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e522a0fc8f6..f00c7460e82 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport module CommandLineUtil + DEFAULT_MODE = 0700 + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -21,6 +23,11 @@ module Gitlab execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) end + def mkdir_p(path) + FileUtils.mkdir_p(path, mode: DEFAULT_MODE) + FileUtils.chmod(DEFAULT_MODE, path) + end + private def tar_with_options(archive:, dir:, options:) @@ -45,7 +52,7 @@ module Gitlab # if we are copying files, create the destination folder destination_folder = File.file?(source) ? File.dirname(destination) : destination - FileUtils.mkdir_p(destination_folder) + mkdir_p(destination_folder) FileUtils.copy_entry(source, destination) true end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index eca6e5b6d51..113895ba22c 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -15,7 +15,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) wait_for_archived_file do decompress_archive diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index bb9d1080330..e6ecd118609 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,6 +1,7 @@ # Model relationships to be included in the project import/export project_tree: - - :labels + - labels: + :priorities - milestones: - :events - issues: @@ -9,7 +10,8 @@ project_tree: - :author - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - snippets: @@ -26,7 +28,8 @@ project_tree: - :merge_request_diff - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - pipelines: @@ -71,6 +74,10 @@ excluded_attributes: - :awardable_id methods: + labels: + - :type + label: + - :type statuses: - :type services: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 0cc10f40087..48c09dafcb6 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -65,11 +65,17 @@ module Gitlab # +value+ existing model to be included in the hash # +parsed_hash+ the original hash def parse_hash(value) + return nil if already_contains_methods?(value) + @attributes_finder.parse(value) do |hash| { include: hash_or_merge(value, hash) } end end + def already_contains_methods?(value) + value.is_a?(Hash) && value.values.detect { |val| val[:methods]} + end + # Adds new model configuration to an existing hash with key +current_key+ # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ # diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 35ff134ea19..7cdba880a93 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -110,13 +110,18 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => restored_project.id), + relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, - user: @user) + user: @user, + project_id: restored_project.id) end relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end + + def parsed_relation_hash(relation_hash) + relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + end end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 9153088e966..2fbf437ec26 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport class ProjectTreeSaver + include Gitlab::ImportExport::CommandLineUtil + attr_reader :full_path def initialize(project:, shared:) @@ -10,7 +12,7 @@ module Gitlab end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0c2cd208bf9..ce31c923feb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -9,25 +9,28 @@ module Gitlab builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze + push_access_levels: 'ProtectedBranch::PushAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze - - FINDER_ATTRIBUTES = %w[title project_id].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze def self.create(*args) new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id') + @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -54,6 +57,8 @@ module Gitlab update_user_references update_project_references + + handle_group_label if group_label? reset_ci_tokens if @relation_name == 'Ci::Trigger' @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff @@ -121,6 +126,20 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end + def group_label? + @relation_hash['type'] == 'GroupLabel' + end + + def handle_group_label + # If there's no group, move the label to a project label + if @relation_hash['group_id'] + @relation_hash['project_id'] = nil + @relation_name = :group_label + else + @relation_hash['type'] = 'ProjectLabel' + end + end + def reset_ci_tokens return unless Gitlab::ImportExport.reset_tokens? @@ -153,7 +172,11 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= begin + Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash) + + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end end def set_st_diffs @@ -165,11 +188,9 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - events = parsed_relation_hash.delete('events') + attribute_hash = attribute_hash_for(['events', 'priorities']) - unless events.blank? - existing_object.assign_attributes(events: events) - end + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? existing_object else @@ -178,14 +199,22 @@ module Gitlab end end + def attribute_hash_for(attributes) + attributes.inject({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + def existing_object @existing_object ||= begin - finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES) + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) existing_object = relation_class.find_or_create_by(finder_hash) # Done in two steps, as MySQL behaves differently than PostgreSQL using # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update(parsed_relation_hash) + existing_object.update!(parsed_relation_hash) existing_object end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d1e33ea8678..48a9a6fa5e2 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -12,7 +12,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - FileUtils.mkdir_p(path_to_repo) + mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks rescue => e diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 331e14021e6..a7028a32570 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -20,7 +20,7 @@ module Gitlab private def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 9b642d740b7..7cf88298642 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,12 +1,14 @@ module Gitlab module ImportExport class VersionSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(shared:) @shared = shared end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 6107420e4dd..1e6722a7bba 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -9,7 +9,7 @@ module Gitlab end def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 1bec6088292..dbc759367eb 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -18,8 +18,8 @@ module Gitlab { title: "enhancement", color: green } ] - labels.each do |label| - project.labels.create(label) + labels.each do |params| + ::Labels::FindOrCreateService.new(project.owner, project, params).execute end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 7b3bbcf6a32..5f67e97fa2a 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -20,13 +20,8 @@ module Gitlab def token Gitlab::Redis.with do |redis| token = redis.get(redis_key) - - if token - redis.expire(redis_key, EXPIRY_TIME) - else - token = Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) - end + token ||= Devise.friendly_token(TOKEN_LENGTH) + redis.set(redis_key, token, ex: EXPIRY_TIME) token end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 5b9cfaeb2f8..24733435a5a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -73,11 +73,7 @@ module Gitlab end def commits - if project.empty_repo? || query.blank? - [] - else - project.repository.find_commits_by_message(query).compact - end + project.repository.find_commits_by_message(query) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 69c4ef721d5..c649da8c426 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -11,13 +11,6 @@ module Gitlab DEFAULT_REDIS_URL = 'redis://localhost:6379' CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) - # To be thread-safe we must be careful when writing the class instance - # variables @_raw_config and @pool. Because @pool depends on @_raw_config we need two - # mutexes to prevent deadlock. - RAW_CONFIG_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :RAW_CONFIG_MUTEX, :POOL_MUTEX - class << self # Do NOT cache in an instance variable. Result may be mutated by caller. def params @@ -31,24 +24,29 @@ module Gitlab end def with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(params) } - end - end + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @pool.with { |redis| yield redis } end + def pool_size + if Sidekiq.server? + # the pool will be used in a multi-threaded context + Sidekiq.options[:concurrency] + 5 + else + # probably this is a Unicorn process, so single threaded + 5 + end + end + def _raw_config return @_raw_config if defined?(@_raw_config) - RAW_CONFIG_MUTEX.synchronize do - begin - @_raw_config = File.read(CONFIG_FILE).freeze - rescue Errno::ENOENT - @_raw_config = false - end + begin + @_raw_config = File.read(CONFIG_FILE).freeze + rescue Errno::ENOENT + @_raw_config = false end + @_raw_config end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 776bbcbb5d0..0d30e1bb92e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,7 +2,7 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze + NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze @@ -10,7 +10,7 @@ module Gitlab def namespace_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.'." \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ end def namespace_name_regex diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 7813091ec7b..82a59a7a87e 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -2,7 +2,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{job['args']}" + Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" yield end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5d33f98e89e..594439a5d4b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -111,7 +111,7 @@ module Gitlab def write_secret bytes = SecureRandom.random_bytes(SECRET_LENGTH) File.open(secret_path, 'w:BINARY', 0600) do |f| - f.chmod(0600) + f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. f.write(Base64.strict_encode64(bytes)) end end diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/.gitkeep +++ /dev/null diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 2214f855200..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,33 @@ namespace :cache do - CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + namespace :clear do + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task :clear => :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: REDIS_CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end + + desc "GitLab | Clear database cache (in the background)" + task db: :environment do + ClearDatabaseCacheWorker.perform_async + end + + task all: [:db, :redis] end + + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake new file mode 100644 index 00000000000..424e7883060 --- /dev/null +++ b/lib/tasks/ce_to_ee_merge_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ce_to_ee_merge_check: :environment do + Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke +end diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/ci/.gitkeep +++ /dev/null diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 5f4a6bbfa35..2ae48a970ce 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -671,7 +671,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -690,7 +690,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -747,7 +747,7 @@ namespace :gitlab do "Check that the information in config/gitlab.yml is correct" ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake new file mode 100644 index 00000000000..47bdb2d32d2 --- /dev/null +++ b/lib/tasks/gitlab/dev.rake @@ -0,0 +1,107 @@ +namespace :gitlab do + namespace :dev do + desc 'Checks if the branch would apply cleanly to EE' + task ce_to_ee_merge_check: :environment do + return if defined?(Gitlab::License) + return unless ENV['CI'] + + ce_repo = ENV['CI_BUILD_REPO'] + ce_branch = ENV['CI_BUILD_REF_NAME'] + + ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' + ee_branch = "#{ce_branch}-ee" + ee_dir = 'gitlab-ee-merge-check' + + puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" + `git clone #{ee_repo} #{ee_dir} --depth 1` + Dir.chdir(ee_dir) do + puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" + `git fetch #{ce_repo} #{ce_branch} --depth 1` + + # Try to merge the current tested branch to EE/master... + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge FETCH_HEAD` + + exit 0 if $?.success? + + # Check if the <branch>-ee branch exists... + puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" + `git rev-parse --verify #{ee_branch}` + + # The <branch>-ee doesn't exist + unless $?.success? + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to the + current EE/master, and no #{ee_branch} branch was detected in + the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + You can create this branch as follows: + + 1. In the EE repo: + $ git fetch origin + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git rebase origin/master + 2. At this point you will likely have conflicts, solve them, and + continue/finish the rebase. Note: You can squash the CE commits + before rebasing. + 3. You can squash all the original #{ce_branch} commits into a + single "Port of #{ce_branch} to EE". + 4. Push your branch to #{ee_repo}: + $ git push origin #{ee_branch} + =================================================================\n + MSG + + exit 1 + end + + # Try to merge the <branch>-ee branch to EE/master... + puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" + `git merge #{ee_branch} master` + + # The <branch>-ee cannot be merged cleanly to EE/master... + unless $?.success? + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to + EE/master, and even though the #{ee_branch} branch exists in the EE + repository, it cannot be merged without conflicts to EE/master. + + Please update the #{ee_branch}, push it again to #{ee_repo}, and + retry this job. + =================================================================\n + MSG + + exit 2 + end + + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge FETCH_HEAD` + exit 0 if $?.success? + + # The <branch>-ee can be merged cleanly to EE/master, but <branch> still + # cannot be merged cleanly to EE/master... + puts + puts <<-MSG.strip_heredoc + ================================================================= + The #{ce_branch} branch cannot be merged without conflicts to EE, and + even though the #{ee_branch} branch exists in the EE repository and + applies cleanly to EE/master, it doesn't prevent conflicts when + merging #{ce_branch} into EE. + + We may be in a complex situation here. + =================================================================\n + MSG + + exit 3 + end + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index bb7eb852f1b..210899882b4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -78,7 +78,7 @@ namespace :gitlab do f.puts "PATH=#{ENV['PATH']}" end - Gitlab::Shell.new.generate_and_link_secret_token + Gitlab::Shell.ensure_secret_token! end desc "GitLab | Setup gitlab-shell" diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake new file mode 100644 index 00000000000..3a16ace60bd --- /dev/null +++ b/lib/tasks/gitlab/users.rake @@ -0,0 +1,11 @@ +namespace :gitlab do + namespace :users do + desc "GitLab | Clear the authentication token for all users" + task clear_all_authentication_tokens: :environment do |t, args| + # Do small batched updates because these updates will be slow and locking + User.select(:id).find_in_batches(batch_size: 100) do |batch| + User.where(id: batch.map(&:id)).update_all(authentication_token: nil) + end + end + end +end |