diff options
Diffstat (limited to 'lib')
66 files changed, 1091 insertions, 325 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 592100a7045..231840148d9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -64,7 +64,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch does not exist") unless @branch + not_found!("Branch") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 340fc5452ab..60b9f5e0ece 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -15,7 +15,7 @@ module API class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url end class Identity < Grape::Entity @@ -170,6 +170,10 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic + + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user]) + end end class MergeRequest < ProjectEntity @@ -183,6 +187,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user]) + end end class MergeRequestChanges < MergeRequest @@ -204,7 +212,7 @@ module API expose :note, as: :body expose :attachment_identifier, as: :attachment expose :author, using: Entities::UserBasic - expose :created_at + expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type # upvote? and downvote? are deprecated, always return false @@ -255,14 +263,19 @@ module API expose :id, :path, :kind end - class ProjectAccess < Grape::Entity + class Member < Grape::Entity expose :access_level - expose :notification_level + expose :notification_level do |member, options| + if member.notification_setting + NotificationSetting.levels[member.notification_setting.level] + end + end end - class GroupAccess < Grape::Entity - expose :access_level - expose :notification_level + class ProjectAccess < Member + end + + class GroupAccess < Member end class ProjectService < Grape::Entity @@ -293,6 +306,7 @@ module API class Label < Grape::Entity expose :name, :color, :description + expose :open_issues_count, :closed_issues_count, :open_merge_requests_count end class Compare < Grape::Entity diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c165de21a75..91e420832f3 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,8 +23,10 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group # Example Request: # POST /groups post do @@ -42,6 +44,28 @@ module API end end + # Update group. Available only for users who can administrate groups. + # + # Parameters: + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # Example Request: + # PUT /groups/:id + put ':id' do + group = find_group(params[:id]) + authorize! :admin_group, group + + attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + + if ::Groups::UpdateService.new(group, current_user, attrs).execute + present group, with: Entities::GroupDetail + else + render_validation_error!(group) + end + end + # Get a single group, with containing projects # # Parameters: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4921ae99e78..5bbf721321d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,7 @@ module API if can?(current_user, :read_group, group) group else - forbidden!("#{current_user.username} lacks sufficient "\ - "access to #{group.name}") + not_found!('Group') end end @@ -241,6 +240,10 @@ module API render_api_error!('413 Request Entity Too Large', 413) end + def not_modified! + render_api_error!('304 Not Modified', 304) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 2200208b946..8cfa1f1556b 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -23,9 +23,11 @@ module API end post "/allowed" do + Gitlab::Metrics.tag_transaction('action', 'Grape#/internal/allowed') + status 200 - actor = + actor = if params[:key_id] Key.find_by(id: params[:key_id]) elsif params[:user_id] @@ -33,7 +35,7 @@ module API end project_path = params[:project] - + # Check for *.wiki repositories. # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1fee1dee1a6..8aa08fd5acc 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -55,7 +55,7 @@ module API issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -92,7 +92,7 @@ module API end issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end # Get a single project issue @@ -105,7 +105,7 @@ module API get ":id/issues/:issue_id" do @issue = user_project.issues.find(params[:issue_id]) not_found! unless can?(current_user, :read_issue, @issue) - present @issue, with: Entities::Issue + present @issue, with: Entities::Issue, current_user: current_user end # Create a new project issue @@ -117,7 +117,7 @@ module API # assignee_id (optional) - The ID of a user to assign issue # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue - # created_at (optional) - The date + # created_at (optional) - Date time string, ISO 8601 formatted # Example Request: # POST /projects/:id/issues post ":id/issues" do @@ -149,7 +149,7 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end @@ -166,12 +166,15 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) + # updated_at (optional) - Date time string, ISO 8601 formatted # Example Request: # PUT /projects/:id/issues/:issue_id put ":id/issues/:issue_id" do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys << :updated_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -189,12 +192,35 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end end + # Move an existing issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # to_project_id (required) - The ID of the new project + # Example Request: + # POST /projects/:id/issues/:issue_id/move + post ':id/issues/:issue_id/move' do + required_attributes! [:to_project_id] + + issue = user_project.issues.find(params[:issue_id]) + new_project = Project.find(params[:to_project_id]) + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: Entities::Issue, current_user: current_user + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + # # Delete a project issue # # Parameters: @@ -208,6 +234,42 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + # Subscribes to a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # POST /projects/:id/issues/:issue_id/subscription + post ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + not_modified! + else + issue.toggle_subscription(current_user) + present issue, with: Entities::Issue, current_user: current_user + end + end + + # Unsubscribes from a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # DELETE /projects/:id/issues/:issue_id/subscription + delete ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + issue.unsubscribe(current_user) + present issue, with: Entities::Issue, current_user: current_user + else + not_modified! + end + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 93052fba06b..7e78609ecb9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -56,7 +56,7 @@ module API end merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user end # Create MR @@ -94,7 +94,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -130,7 +130,7 @@ module API authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Show MR commits @@ -162,7 +162,7 @@ module API merge_request = user_project.merge_requests. find(params[:merge_request_id]) authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end # Update MR @@ -204,7 +204,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -246,7 +246,7 @@ module API execute(merge_request) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Cancel Merge if Merge When build succeeds is enabled @@ -325,7 +325,43 @@ module API get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user + end + + # Subscribes to a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # POST /projects/:id/issues/:merge_request_id/subscription + post "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + not_modified! + else + merge_request.toggle_subscription(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + end + end + + # Unsubscribes from a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # DELETE /projects/:id/merge_requests/:merge_request_id/subscription + delete "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + merge_request.unsubscribe(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + else + not_modified! + end end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index c5cd73943fb..84b4d4cdd6d 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -3,17 +3,35 @@ module API class Milestones < Grape::API before { authenticate! } + helpers do + def filter_milestones_state(milestones, state) + case state + when 'active' then milestones.active + when 'closed' then milestones.closed + else milestones + end + end + end + resource :projects do # Get a list of project milestones # # Parameters: - # id (required) - The ID of a project + # id (required) - The ID of a project + # state (optional) - Return "active" or "closed" milestones # Example Request: # GET /projects/:id/milestones + # GET /projects/:id/milestones?iid=42 + # GET /projects/:id/milestones?state=active + # GET /projects/:id/milestones?state=closed get ":id/milestones" do authorize! :read_milestone, user_project - present paginate(user_project.milestones), with: Entities::Milestone + milestones = user_project.milestones + milestones = filter_milestones_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + + present paginate(milestones), with: Entities::Milestone end # Get a single project milestone @@ -87,7 +105,7 @@ module API authorize! :read_milestone, user_project @milestone = user_project.milestones.find(params[:milestone_id]) - present paginate(@milestone.issues), with: Entities::Issue + present paginate(@milestone.issues), with: Entities::Issue, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..71a53e6f0d6 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -61,6 +61,7 @@ module API # id (required) - The ID of a project # noteable_id (required) - The ID of an issue or snippet # body (required) - The content of a note + # created_at (optional) - The date # Example Request: # POST /projects/:id/issues/:noteable_id/notes # POST /projects/:id/snippets/:noteable_id/notes @@ -73,6 +74,10 @@ module API noteable_id: params[noteable_id_str] } + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end + @note = ::Notes::CreateService.new(user_project, current_user, opts).execute if @note.valid? @@ -112,6 +117,23 @@ module API end end + # Delete a +noteable+ note + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue, MR, or snippet + # node_id (required) - The ID of a note + # Example Request: + # DELETE /projects/:id/issues/:noteable_id/notes/:note_id + # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id + delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DeleteService.new(user_project, current_user).execute(note) + + present note, with: Entities::Note + end end end end diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index c756bb479fc..4aefdf319c6 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -93,12 +93,17 @@ module API # Example Request: # DELETE /projects/:id/members/:user_id delete ":id/members/:user_id" do - authorize! :admin_project, user_project project_member = user_project.project_members.find_by(user_id: params[:user_id]) - unless project_member.nil? - project_member.destroy - else + + unless current_user.can?(:admin_project, user_project) || + current_user.can?(:destroy_project_member, project_member) + forbidden! + end + + if project_member.nil? { message: "Access revoked", id: params[:user_id].to_i } + else + project_member.destroy end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 24b31005475..cc2c7a0c503 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -272,6 +272,40 @@ module API present user_project, with: Entities::Project end + # Star project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/star + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + end + end + + # Unstar project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id/star + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + else + not_modified! + end + end + # Remove project # # Parameters: diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0d0f0d4616d..62161aadb9a 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -98,7 +98,6 @@ module API authorize! :download_code, user_project begin - RepositoryArchiveCacheWorker.perform_async header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) rescue not_found!('File') diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 2d8a9e51bb9..d1a10479e44 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -16,6 +16,20 @@ module API with: Entities::RepoTag, project: user_project end + # Get a single repository tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # GET /projects/:id/repository/tags/:tag_name + get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + present tag, with: Entities::RepoTag, project: user_project + end + # Create tag # # Parameters: diff --git a/lib/api/users.rb b/lib/api/users.rb index 13ab17c6904..ea6fa2dc8a8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -11,6 +11,10 @@ module API # GET /users?search=Admin # GET /users?username=root get do + unless can?(current_user, :read_users_list, nil) + render_api_error!("Not authorized.", 403) + end + if params[:username].present? @users = User.where(username: params[:username]) else @@ -36,10 +40,12 @@ module API get ":id" do @user = User.find(params[:id]) - if current_user.is_admin? + if current_user && current_user.is_admin? present @user, with: Entities::UserFull - else + elsif can?(current_user, :read_user, @user) present @user, with: Entities::User + else + render_api_error!("User not found.", 404) end end @@ -58,6 +64,7 @@ module API # extern_uid - External authentication provider UID # provider - External provider # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # confirm - Require user confirmation - true (default) or false @@ -67,7 +74,7 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) user = User.build_user(attrs) @@ -106,6 +113,7 @@ module API # website_url - Website url # projects_limit - Limit projects each user can create # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # external - Flags the user as external - true or false(default) @@ -114,7 +122,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 4fc3443ac68..5f8ff01b0a9 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -14,19 +14,29 @@ class AwardEmoji food_drink: "Food" }.with_indifferent_access + CATEGORY_ALIASES = { + symbols: "objects_symbols", + foods: "food_drink", + travel: "travel_places" + }.with_indifferent_access + def self.normilize_emoji_name(name) aliases[name] || name end def self.emoji_by_category unless @emoji_by_category - @emoji_by_category = {} + @emoji_by_category = Hash.new { |h, key| h[key] = [] } emojis.each do |emoji_name, data| data["name"] = emoji_name - @emoji_by_category[data["category"]] ||= [] - @emoji_by_category[data["category"]] << data + # Skip Fitzpatrick(tone) modifiers + next if data["category"] == "modifier" + + category = CATEGORY_ALIASES[data["category"]] || data["category"] + + @emoji_by_category[category] << data end @emoji_by_category = @emoji_by_category.sort.to_h diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index f21dbef216c..b8962379cb5 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -119,7 +119,7 @@ module Banzai elsif element_node?(node) yield_valid_link(node) do |link, text| - if ref_pattern && link =~ /\A#{ref_pattern}/ + if ref_pattern && link =~ /\A#{ref_pattern}\z/ replace_link_node_with_href(node, link) do object_link_filter(link, ref_pattern, link_text: text) end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 7ce26db1b90..d08267a9d6c 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -118,7 +118,7 @@ module Banzai end if path - content_tag(:img, nil, src: path) + content_tag(:img, nil, src: path, class: 'gfm') end end @@ -144,12 +144,18 @@ module Banzai # if it is not. def process_page_link_tag(parts) if parts.size == 1 - url = parts[0].strip + reference = parts[0].strip else - name, url = *parts.compact.map(&:strip) + name, reference = *parts.compact.map(&:strip) end - content_tag(:a, name || url, href: url) + if url?(reference) + href = reference + else + href = ::File.join(project_wiki_base_path, reference) + end + + content_tag(:a, name || reference, href: href, class: 'gfm') end def project_wiki diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb new file mode 100644 index 00000000000..ccd106860bd --- /dev/null +++ b/lib/banzai/filter/image_link_filter.rb @@ -0,0 +1,27 @@ +module Banzai + module Filter + # HTML filter that wraps links around inline images. + class ImageLinkFilter < HTML::Pipeline::Filter + + # Find every image that isn't already wrapped in an `a` tag, create + # a new node (a link to the image source), copy the image as a child + # of the anchor, and then replace the img with the link-wrapped version. + def call + doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + + link = doc.document.create_element( + 'a', + class: 'no-attachment-icon', + href: img['src'], + target: '_blank' + ) + + link.children = img.clone + img.replace(link) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb new file mode 100644 index 00000000000..06d10c98501 --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -0,0 +1,56 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" relative links to files in a repository. + # + # Context options: + # :project_wiki + class WikiLinkFilter < HTML::Pipeline::Filter + + def call + return doc unless project_wiki? + + doc.search('a:not(.gfm)').each do |el| + process_link_attr el.attribute('href') + end + + doc + end + + protected + + def project_wiki? + !context[:project_wiki].nil? + end + + def process_link_attr(html_attr) + return if html_attr.blank? || file_reference?(html_attr) + + uri = URI(html_attr.value) + if uri.relative? && uri.path.present? + html_attr.value = rebuild_wiki_uri(uri).to_s + end + rescue URI::Error + # noop + end + + def rebuild_wiki_uri(uri) + uri.path = ::File.join(project_wiki_base_path, uri.path) + uri + end + + def file_reference?(html_attr) + !File.extname(html_attr.value).blank? + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8cd4b50e65a..ed3cfd6b023 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::UploadLinkFilter, + Filter::ImageLinkFilter, Filter::EmojiFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 0b5a9e0b2b8..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -2,8 +2,10 @@ module Banzai module Pipeline class WikiPipeline < FullPipeline def self.filters - @filters ||= super.insert_after(Filter::TableOfContentsFilter, - Filter::GollumTagsFilter) + @filters ||= begin + super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) + .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ae714c87dc5..c14a9c4c722 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -19,8 +19,10 @@ module Banzai cache_key = full_cache_key(cache_key, context[:pipeline]) if cache_key - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + Gitlab::Metrics.measure(:banzai_cached_render) do + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) + end end else cacheless_render(text, context) @@ -64,13 +66,15 @@ module Banzai private def self.cacheless_render(text, context = {}) - result = render_result(text, context) + Gitlab::Metrics.measure(:banzai_cacheless_render) do + result = render_result(text, context) - output = result[:output] - if output.respond_to?(:to_html) - output.to_html - else - output.to_s + output = result[:output] + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b7209c14148..ff9887cba1e 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,12 +4,12 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies] + :dependencies, :before_script, :after_script, :variables] - attr_reader :before_script, :image, :services, :variables, :path, :cache + attr_reader :before_script, :after_script, :image, :services, :path, :cache def initialize(config, path = nil) @config = YAML.safe_load(config, [Symbol], [], true) @@ -40,10 +40,22 @@ module Ci @stages || DEFAULT_STAGES end + def global_variables + @variables + end + + def job_variables(name) + job = @jobs[name.to_sym] + return [] unless job + + job.fetch(:variables, []) + end + private def initial_parsing @before_script = @config[:before_script] || [] + @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @stages = @config[:stages] || @config[:types] @@ -72,7 +84,7 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], @@ -85,23 +97,32 @@ module Ci artifacts: job[:artifacts], cache: job[:cache] || @cache, dependencies: job[:dependencies], + after_script: job[:after_script] || @after_script, }.compact } end - def normalize_script(script) - if script.is_a? Array - script.join("\n") - else - script + def validate! + validate_global! + + @jobs.each do |name, job| + validate_job!(name, job) end + + true end - def validate! + private + + def validate_global! unless validate_array_of_strings(@before_script) raise ValidationError, "before_script should be an array of strings" end + unless @after_script.nil? || validate_array_of_strings(@after_script) + raise ValidationError, "after_script should be an array of strings" + end + unless @image.nil? || @image.is_a?(String) raise ValidationError, "image should be a string" end @@ -115,43 +136,39 @@ module Ci end unless @variables.nil? || validate_variables(@variables) - raise ValidationError, "variables should be a map of key-valued strings" + raise ValidationError, "variables should be a map of key-value strings" end - if @cache - if @cache[:key] && !validate_string(@cache[:key]) - raise ValidationError, "cache:key parameter should be a string" - end - - if @cache[:untracked] && !validate_boolean(@cache[:untracked]) - raise ValidationError, "cache:untracked parameter should be an boolean" - end + validate_global_cache! if @cache + end - if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) - raise ValidationError, "cache:paths parameter should be an array of strings" - end + def validate_global_cache! + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" end - @jobs.each do |name, job| - validate_job!(name, job) + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" end - true + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end end def validate_job!(name, job) validate_job_name!(name) validate_job_keys!(name, job) validate_job_types!(name, job) + validate_job_script!(name, job) validate_job_stage!(name, job) if job[:stage] + validate_job_variables!(name, job) if job[:variables] validate_job_cache!(name, job) if job[:cache] validate_job_artifacts!(name, job) if job[:artifacts] validate_job_dependencies!(name, job) if job[:dependencies] end - private - def validate_job_name!(name) if name.blank? || !validate_string(name) raise ValidationError, "job name should be non-empty string" @@ -167,10 +184,6 @@ module Ci end def validate_job_types!(name, job) - if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name} job: script should be a string or an array of a strings" - end - if job[:image] && !validate_string(job[:image]) raise ValidationError, "#{name} job: image should be a string" end @@ -200,12 +213,33 @@ module Ci end end + def validate_job_script!(name, job) + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" + end + + if job[:before_script] && !validate_array_of_strings(job[:before_script]) + raise ValidationError, "#{name} job: before_script should be an array of strings" + end + + if job[:after_script] && !validate_array_of_strings(job[:after_script]) + raise ValidationError, "#{name} job: after_script should be an array of strings" + end + end + def validate_job_stage!(name, job) unless job[:stage].is_a?(String) && job[:stage].in?(stages) raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" end end + def validate_job_variables!(name, job) + unless validate_variables(job[:variables]) + raise ValidationError, + "#{name} job: variables should be a map of key-value strings" + end + end + def validate_job_cache!(name, job) if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 6108697bc20..7479e729db1 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,7 @@ require 'gitlab/git' module Gitlab + def self.com? + Gitlab.config.gitlab.url == 'https://gitlab.com' + end end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index b9bb6e76081..5e2fb863a8f 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -54,19 +54,6 @@ module Gitlab "#{path}.git", "#{new_path}.git"]) end - # Update HEAD for repository - # - # path - project path with namespace - # branch - repository branch name - # - # Ex. - # update_repository_head("gitlab/gitlab-ci", "3-1-stable") - # - def update_repository_head(path, branch) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'update-head', - "#{path}.git", branch]) - end - # Fork repository to new namespace # # path - project path with namespace @@ -92,33 +79,6 @@ module Gitlab 'rm-project', "#{name}.git"]) end - # Add repository branch from passed ref - # - # path - project path with namespace - # branch_name - new branch name - # ref - HEAD for new branch - # - # Ex. - # add_branch("gitlab/gitlab-ci", "4-0-stable", "master") - # - def add_branch(path, branch_name, ref) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'create-branch', - "#{path}.git", branch_name, ref]) - end - - # Remove repository branch - # - # path - project path with namespace - # branch_name - branch name to remove - # - # Ex. - # rm_branch("gitlab/gitlab-ci", "4-0-stable") - # - def rm_branch(path, branch_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-branch', - "#{path}.git", branch_name]) - end - # Add repository tag from passed ref # # path - project path with namespace @@ -137,19 +97,6 @@ module Gitlab Gitlab::Utils.system_silent(cmd) end - # Remove repository tag - # - # path - project path with namespace - # tag_name - tag name to remove - # - # Ex. - # rm_tag("gitlab/gitlab-ci", "v4.0") - # - def rm_tag(path, tag_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-tag', - "#{path}.git", tag_name]) - end - # Gc repository # # path - project path with namespace diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb index 28a2391dbf8..e5e9fab3f5c 100644 --- a/lib/gitlab/badge/build.rb +++ b/lib/gitlab/badge/build.rb @@ -4,14 +4,15 @@ module Gitlab # Build badge # class Build + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + def initialize(project, ref) + @project, @ref = project, ref @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) end - def to_s - @image[:name].sub(/\.svg$/, '') - end - def type 'image/svg+xml' end @@ -19,6 +20,27 @@ module Gitlab def data File.read(@image[:path]) end + + def to_s + @image[:name].sub(/\.svg$/, '') + end + + def to_html + link_to(image_tag(image_url, alt: 'build status'), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end end end end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index d88a6eaac6b..9bb507b5edd 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -5,6 +5,17 @@ module Gitlab attr_reader :consumer, :api + def self.from_project(project) + import_data_credentials = project.import_data.credentials if project.import_data + if import_data_credentials && import_data_credentials[:bb_session] + token = import_data_credentials[:bb_session][:bitbucket_access_token] + token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] + new(token, token_secret) + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end + end + def initialize(access_token = nil, access_token_secret = nil) @consumer = ::OAuth::Consumer.new( config.app_id, @@ -54,7 +65,7 @@ module Gitlab def issues(project_identifier) all_issues = [] offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket + per_page = 50 # Maximum number allowed by Bitbucket index = 0 begin @@ -120,7 +131,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} + Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } end def bitbucket_options diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 46e51a4bf6d..7beaecd1cf0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -5,10 +5,7 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb index f4dd393ad29..e03c3155b3e 100644 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ b/lib/gitlab/bitbucket_import/key_deleter.rb @@ -6,10 +6,7 @@ module Gitlab def initialize(project) @project = project @current_user = project.creator - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) end def execute diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 03aac1a025a..941f818b847 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -23,7 +23,8 @@ module Gitlab import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", ).execute - project.create_import_data(data: { "bb_session" => session_data } ) + project.create_or_update_import_data(credentials: { bb_session: session_data }) + project end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 1acc22fe5bf..f44d1b3a44e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -34,7 +34,8 @@ module Gitlab max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, two_factor_grace_period: 48, - akismet_enabled: false + akismet_enabled: false, + repository_checks_enabled: true, ) end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index c73eca832d7..ffe49364379 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -43,18 +43,15 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + Gitlab::Redis.with do |redis| + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end end # No #cancel method. See comments above! private - def redis - # Maybe someday we want to use a connection pool... - @redis ||= Redis.new(url: Gitlab::RedisConfig.url) - end - def redis_key "gitlab:exclusive_lease:#{@key}" end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index db580b5e578..501d5a95547 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -8,17 +8,17 @@ module Gitlab import_data = project.import_data.try(:data) repo_data = import_data['repo'] if import_data - @repo = FogbugzImport::Repository.new(repo_data) - - @known_labels = Set.new + if repo_data + @repo = FogbugzImport::Repository.new(repo_data) + @known_labels = Set.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute return true unless repo.valid? - - data = project.import_data.try(:data) - - client = Gitlab::FogbugzImport::Client.new(token: data['fb_session']['token'], uri: data['fb_session']['uri']) + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @categories = client.categories @@ -30,6 +30,10 @@ module Gitlab private + def fb_session + @import_data_credentials ||= project.import_data.credentials[:fb_session] if project.import_data && project.import_data.credentials + end + def user_map @user_map ||= begin user_map = Hash.new @@ -236,9 +240,8 @@ module Gitlab end def build_attachment_url(rel_url) - data = project.import_data.try(:data) - uri = data['fb_session']['uri'] - token = data['fb_session']['token'] + uri = fb_session[:uri] + token = fb_session[:token] "#{uri}/#{rel_url}&token=#{token}" end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index e0163499e30..3840765db87 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -24,13 +24,7 @@ module Gitlab import_url: Project::UNKNOWN_IMPORT_URL ).execute - project.create_import_data( - data: { - 'repo' => repo.raw_data, - 'user_map' => user_map, - 'fb_session' => fb_session - } - ) + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session }) project end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 172c5441e36..0f9e3ee14ee 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -7,18 +7,45 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - github_session = import_data["github_session"] if import_data - @client = Client.new(github_session["github_access_token"]) - @formatter = Gitlab::ImportFormatter.new + if import_data_credentials + @client = Client.new(import_data_credentials[:user]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute - import_issues && import_pull_requests && import_wiki + import_labels && import_milestones && import_issues && + import_pull_requests && import_wiki end private + def import_data_credentials + @import_data_credentials ||= project.import_data.credentials if project.import_data + end + + def import_labels + client.labels(project.import_source).each do |raw_data| + Label.create!(LabelFormatter.new(project, raw_data).attributes) + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + + def import_milestones + client.list_milestones(project.import_source, state: :all).each do |raw_data| + Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, @@ -27,6 +54,7 @@ module Gitlab if gh_issue.valid? issue = Issue.create!(gh_issue.attributes) + apply_labels(gh_issue.number, issue) if gh_issue.has_comments? import_comments(gh_issue.number, issue) @@ -49,6 +77,7 @@ module Gitlab merge_request = MergeRequest.new(pull_request.attributes) if merge_request.save + apply_labels(pull_request.number, merge_request) import_comments(pull_request.number, merge_request) import_comments_on_diff(pull_request.number, merge_request) end @@ -60,6 +89,18 @@ module Gitlab raise Projects::ImportService::Error, e.message end + def apply_labels(number, issuable) + issue = client.issue(project.import_source, number) + + if issue.labels.count > 0 + label_ids = issue.labels.map do |raw| + Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) + end + + issuable.update_attribute(:label_ids, label_ids) + end + end + def import_comments(issue_number, noteable) comments = client.issue_comments(project.import_source, issue_number) create_comments(comments, noteable) diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 1e3ba44f27c..c8173913b4e 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -3,7 +3,9 @@ module Gitlab class IssueFormatter < BaseFormatter def attributes { + iid: number, project: project, + milestone: milestone, title: raw_data.title, description: description, state: state, @@ -54,6 +56,12 @@ module Gitlab @formatter.author_line(author) + body end + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end + end + def state raw_data.state == 'closed' ? 'closed' : 'opened' end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb new file mode 100644 index 00000000000..c2b9d40b511 --- /dev/null +++ b/lib/gitlab/github_import/label_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class LabelFormatter < BaseFormatter + def attributes + { + project: project, + title: title, + color: color + } + end + + private + + def color + "##{raw_data.color}" + end + + def title + raw_data.name + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb new file mode 100644 index 00000000000..e91a7e328cf --- /dev/null +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -0,0 +1,48 @@ +module Gitlab + module GithubImport + class MilestoneFormatter < BaseFormatter + def attributes + { + iid: number, + project: project, + title: title, + description: description, + due_date: due_date, + state: state, + created_at: created_at, + updated_at: updated_at + } + end + + private + + def number + raw_data.number + end + + def title + raw_data.title + end + + def description + raw_data.description + end + + def due_date + raw_data.due_on + end + + def state + raw_data.state == 'closed' ? 'closed' : 'active' + end + + def created_at + raw_data.created_at + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 474927069a5..f4221003db5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -23,9 +23,6 @@ module Gitlab import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute - - project.create_import_data(data: { "github_session" => session_data } ) - project end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 4e507b090e8..d21b942ad4b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -3,6 +3,7 @@ module Gitlab class PullRequestFormatter < BaseFormatter def attributes { + iid: number, title: raw_data.title, description: description, source_project: source_project, @@ -10,6 +11,7 @@ module Gitlab target_project: target_project, target_branch: target_branch.name, state: state, + milestone: milestone, author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, @@ -57,6 +59,12 @@ module Gitlab formatter.author_line(author) + body end + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end + end + def source_project project end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 850b73244c6..96717b42bae 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,10 +5,13 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - gitlab_session = import_data["gitlab_session"] if import_data - @client = Client.new(gitlab_session["gitlab_access_token"]) - @formatter = Gitlab::ImportFormatter.new + credentials = import_data + if credentials && credentials[:password] + @client = Client.new(credentials[:password]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 7baaadb813c..77c33db4b59 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -23,7 +23,6 @@ module Gitlab import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - project.create_import_data(data: { "gitlab_session" => session_data } ) project end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb new file mode 100644 index 00000000000..5ebaad6ca6e --- /dev/null +++ b/lib/gitlab/gon_helper.rb @@ -0,0 +1,17 @@ +module Gitlab + module GonHelper + def add_gon_variables + gon.api_version = API::API.version + gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_issues_tracker = Project.new.default_issue_tracker.to_param + gon.max_file_size = current_application_settings.max_attachment_size + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + + if current_user + gon.current_user_id = current_user.id + gon.api_token = current_user.private_token + end + end + end +end diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 87821c23460..0abb7a64c17 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -24,12 +24,7 @@ module Gitlab import_url: repo.import_url ).execute - project.create_import_data( - data: { - "repo" => repo.raw_data, - "user_map" => user_map - } - ) + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }) project end diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/import_url.rb new file mode 100644 index 00000000000..d23b013c1f5 --- /dev/null +++ b/lib/gitlab/import_url.rb @@ -0,0 +1,41 @@ +module Gitlab + class ImportUrl + def initialize(url, credentials: nil) + @url = URI.parse(URI.encode(url)) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index da4435c7308..f2b649e50a2 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -33,7 +33,10 @@ module Gitlab def allowed? if ldap_user - return true unless ldap_config.active_directory + unless ldap_config.active_directory + user.activate if user.ldap_blocked? + return true + end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 88a265c6af2..484970c5a10 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -70,6 +70,50 @@ module Gitlab value.to_s.gsub('=', '\\=') end + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def self.measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def self.tag_transaction(name, value) + trans = current_transaction + + trans.add_tag(name, value) if trans + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -81,5 +125,11 @@ module Gitlab new(udp: { host: host, port: port }) end end + + private + + def self.current_transaction + Transaction.current + end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index face1921d2e..708ef79f304 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -11,6 +11,8 @@ module Gitlab module Instrumentation SERIES = 'method_calls' + PROXY_IVAR = :@__gitlab_instrumentation_proxy + def self.configure yield self end @@ -91,6 +93,18 @@ module Gitlab end end + # Returns true if a module is instrumented. + # + # mod - The module to check + def self.instrumented?(mod) + mod.instance_variable_defined?(PROXY_IVAR) + end + + # Returns the proxy module (if any) of `mod`. + def self.proxy_module(mod) + mod.instance_variable_get(PROXY_IVAR) + end + # Instruments a method. # # type - The type (:class or :instance) of method to instrument. @@ -99,9 +113,8 @@ module Gitlab def self.instrument(type, mod, name) return unless Metrics.enabled? - name = name.to_sym - alias_name = :"_original_#{name}" - target = type == :instance ? mod : mod.singleton_class + name = name.to_sym + target = type == :instance ? mod : mod.singleton_class if type == :instance target = mod @@ -113,6 +126,12 @@ module Gitlab method = mod.method(name) end + unless instrumented?(target) + target.instance_variable_set(PROXY_IVAR, Module.new) + end + + proxy_module = self.proxy_module(target) + # Some code out there (e.g. the "state_machine" Gem) checks the arity of # a method to make sure it only passes arguments when the method expects # any. If we were to always overwrite a method to take an `*args` @@ -125,17 +144,13 @@ module Gitlab args_signature = '*args, &block' end - send_signature = "__send__(#{alias_name.inspect}, #{args_signature})" - - target.class_eval <<-EOF, __FILE__, __LINE__ + 1 - alias_method #{alias_name.inspect}, #{name.inspect} - + proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) trans = Gitlab::Metrics::Instrumentation.transaction if trans start = Time.now - retval = #{send_signature} + retval = super duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold @@ -148,10 +163,12 @@ module Gitlab retval else - #{send_signature} + super end end EOF + + target.prepend(proxy_module) end # Small layer of indirection to make it easier to stub out the current diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 7ea9555cc8c..1cd1ca30f70 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric + JITTER_RANGE = 0.000001..0.001 + attr_reader :series, :values, :tags, :created_at # series - The name of the series (as a String) to store the metric in. @@ -16,11 +18,29 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash + # InfluxDB overwrites an existing point if a new point has the same + # series, tag set, and timestamp. In a highly concurrent environment + # this means that using the number of seconds since the Unix epoch is + # inevitably going to collide with another timestamp. For example, two + # Rails requests processed by different processes may end up generating + # metrics using the _exact_ same timestamp (in seconds). + # + # Due to the way InfluxDB is set up there's no solution to this problem, + # all we can do is lower the amount of collisions. We do this by using + # Time#to_f which returns the seconds as a Float providing greater + # accuracy. We then add a small random value that is large enough to + # distinguish most timestamps but small enough to not alter the amount + # of seconds. + # + # See https://gitlab.com/gitlab-com/operations/issues/175 for more + # information. + time = @created_at.to_f + rand(JITTER_RANGE) + { series: @series, tags: @tags, values: @values, - timestamp: @created_at.to_i * 1_000_000_000 + timestamp: (time * 1_000_000_000).to_i } end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 8008b3bc895..96cad941d5c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,7 @@ module Gitlab return unless current_transaction current_transaction.increment(:sql_duration, event.duration) + current_transaction.increment(:sql_count, 1) end private diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb new file mode 100644 index 00000000000..49e5f86e6e6 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -0,0 +1,39 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in Rails cache calls + class RailsCache < ActiveSupport::Subscriber + attach_to :active_support + + def cache_read(event) + increment(:cache_read_duration, event.duration) + end + + def cache_write(event) + increment(:cache_write_duration, event.duration) + end + + def cache_delete(event) + increment(:cache_delete_duration, event.duration) + end + + def cache_exist?(event) + increment(:cache_exists_duration, event.duration) + end + + def increment(key, duration) + return unless current_transaction + + current_transaction.increment(:cache_duration, duration) + current_transaction.increment(key, duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 83371265278..a7d183b2f94 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -30,6 +30,17 @@ module Gitlab 0 end end + + # THREAD_CPUTIME is not supported on OS X + if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) + def self.cpu_time + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + end + else + def self.cpu_time + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + end + end end end end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index 18523e0aefe..8bdc89a7751 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -59,8 +59,7 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } - base_data[:object_attributes][:url] = - Gitlab::UrlBuilder.new(:note).build(note.id) + base_data[:object_attributes][:url] = Gitlab::UrlBuilder.build(note) base_data end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 832fb08a526..356e96fcbab 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -54,6 +54,12 @@ module Gitlab @user ||= build_new_user end + if external_provider? && @user + @user.external = true + elsif @user + @user.external = false + end + @user end @@ -113,6 +119,10 @@ module Gitlab end end + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index 97d1edab9c1..67622f321a6 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -36,11 +36,12 @@ module Gitlab commit.hook_attrs(with_changed_files: true) end - type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data data = { object_kind: type, + event_name: type, before: oldrev, after: newrev, ref: ref, diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..5c352c96de5 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,50 @@ +module Gitlab + class Redis + CACHE_NAMESPACE = 'cache:gitlab' + SESSION_NAMESPACE = 'session:gitlab' + SIDEKIQ_NAMESPACE = 'resque:gitlab' + + attr_reader :url + + # To be thread-safe we must be careful when writing the class instance + # variables @url and @pool. Because @pool depends on @url we need two + # mutexes to prevent deadlock. + URL_MUTEX = Mutex.new + POOL_MUTEX = Mutex.new + private_constant :URL_MUTEX, :POOL_MUTEX + + def self.url + @url || URL_MUTEX.synchronize { @url = new.url } + end + + def self.with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(url: url) } + end + end + @pool.with { |redis| yield redis } + end + + def self.redis_store_options + url = new.url + redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb deleted file mode 100644 index 4949c6db539..00000000000 --- a/lib/gitlab/redis_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - class RedisConfig - attr_reader :url - - def self.url - new.url - end - - def self.redis_store_options - url = new.url - redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) - if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path - end - redis_config_hash - end - - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) - - @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] - end - end - end -end diff --git a/lib/gitlab/repository_check_logger.rb b/lib/gitlab/repository_check_logger.rb new file mode 100644 index 00000000000..485b596ca57 --- /dev/null +++ b/lib/gitlab/repository_check_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class RepositoryCheckLogger < Gitlab::Logger + def self.file_name_noext + 'repocheck' + end + end +end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb new file mode 100644 index 00000000000..32c1c9ec5bb --- /dev/null +++ b/lib/gitlab/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Saml + class AuthHash < Gitlab::OAuth::AuthHash + + def groups + get_raw(Gitlab::Saml::Config.groups) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + + end + end +end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb new file mode 100644 index 00000000000..0f40c00f547 --- /dev/null +++ b/lib/gitlab/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Saml + class Config + + class << self + def options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + + end + end +end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index b1e30110ef5..dba4bbfc899 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -18,7 +18,7 @@ module Gitlab @user ||= find_or_create_ldap_user end - if auto_link_saml_enabled? + if auto_link_saml_user? @user ||= find_by_email end @@ -26,6 +26,16 @@ module Gitlab @user ||= build_new_user end + if external_users_enabled? && @user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + @user.external = false + else + @user.external = true + end + end + @user end @@ -37,11 +47,24 @@ module Gitlab end end + def changed? + return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + protected - def auto_link_saml_enabled? + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end + + def external_users_enabled? + !Gitlab::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) + end end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f301d42939d..f1943222edf 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -4,50 +4,58 @@ module Gitlab include GitlabRoutingHelper include ActionView::RecordIdentifier - def initialize(type) - @type = type - end + attr_reader :object - def build(id) - case @type - when :issue - build_issue_url(id) - when :merge_request - build_merge_request_url(id) - when :note - build_note_url(id) + def self.build(object) + new(object).url + end + def url + case object + when Commit + commit_url + when Issue + issue_url(object) + when MergeRequest + merge_request_url(object) + when Note + note_url + else + raise NotImplementedError.new("No URL builder defined for #{object.class}") end end private - def build_issue_url(id) - issue = Issue.find(id) - issue_url(issue) + def initialize(object) + @object = object end - def build_merge_request_url(id) - merge_request = MergeRequest.find(id) - merge_request_url(merge_request) + def commit_url(opts = {}) + return '' if object.project.nil? + + namespace_project_commit_url({ + namespace_id: object.project.namespace, + project_id: object.project, + id: object.id + }.merge!(opts)) end - def build_note_url(id) - note = Note.find(id) - if note.for_commit? - namespace_project_commit_url(namespace_id: note.project.namespace, - id: note.commit_id, - project_id: note.project, - anchor: dom_id(note)) - elsif note.for_issue? - issue = Issue.find(note.noteable_id) - issue_url(issue, anchor: dom_id(note)) - elsif note.for_merge_request? - merge_request = MergeRequest.find(note.noteable_id) - merge_request_url(merge_request, anchor: dom_id(note)) - elsif note.for_snippet? - snippet = Snippet.find(note.noteable_id) - project_snippet_url(snippet, anchor: dom_id(note)) + def note_url + if object.for_commit? + commit_url(id: object.commit_id, anchor: dom_id(object)) + + elsif object.for_issue? + issue = Issue.find(object.noteable_id) + issue_url(issue, anchor: dom_id(object)) + + elsif object.for_merge_request? + merge_request = MergeRequest.find(object.noteable_id) + merge_request_url(merge_request, anchor: dom_id(object)) + + elsif object.for_snippet? + snippet = Snippet.find(object.noteable_id) + project_snippet_url(snippet, anchor: dom_id(object)) end end end diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci deleted file mode 100644 index bf05edfd780..00000000000 --- a/lib/support/nginx/gitlab_ci +++ /dev/null @@ -1,29 +0,0 @@ -# GITLAB CI -server { - listen 80 default_server; # e.g., listen 192.168.1.1:80; - server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com; - - access_log /var/log/nginx/gitlab_ci_access.log; - error_log /var/log/nginx/gitlab_ci_error.log; - - # expose API to fix runners - location /api { - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - - # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN - resolver 8.8.8.8 8.8.4.4; - proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # redirect all other CI requests - location / { - return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # adjust this to match the largest build log your runners might submit, - # set to 0 to disable limit - client_max_body_size 10m; -}
\ No newline at end of file diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 51e746ef923..2214f855200 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -4,18 +4,19 @@ namespace :cache do desc "GitLab | Clear redis cache" task :clear => :environment do - redis = Redis.new(url: Gitlab::RedisConfig.url) - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 7ec00a898fd..030ee8bafcb 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -5,12 +5,23 @@ namespace :gemojione do require 'json' dir = Gemojione.index.images_path + digests = [] + aliases = Hash.new { |hash, key| hash[key] = [] } + aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - digests = AwardEmoji.emojis.map do |name, emoji_hash| + JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| + aliases[real_name] << alias_name + end + + AwardEmoji.emojis.map do |name, emoji_hash| fpath = File.join(dir, "#{emoji_hash['unicode']}.png") digest = Digest::SHA256.file(fpath).hexdigest - { name: name, unicode: emoji_hash['unicode'], digest: digest } + digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + + aliases[name].each do |alias_name| + digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + end end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 4cbccf2ca89..48baecfd2a2 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -14,7 +14,7 @@ namespace :gitlab do puts "" end - Rake::Task["db:setup"].invoke + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke |