diff options
Diffstat (limited to 'lib')
118 files changed, 1897 insertions, 1645 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index fe1bf8a4816..7834262d612 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -53,5 +53,6 @@ module API mount Settings mount Keys mount Tags + mount Triggers end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 2c0596c9dfb..1162271f5fc 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -53,7 +53,7 @@ module API name = params[:name] || params[:context] status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(commit: ci_commit, user: current_user) + status ||= GenericCommitStatus.new(project: @project, commit: ci_commit, user: current_user) status.update(attrs) case params[:state].to_s diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 3da6bc415d6..f8511ac5f5c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -45,7 +45,8 @@ module API class ProjectHook < Hook expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :enable_ssl_verification + expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :enable_ssl_verification end class ForkedFromProject < Grape::Entity @@ -63,11 +64,13 @@ module API expose :name, :name_with_namespace expose :path, :path_with_namespace expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :shared_runners_enabled expose :creator_id expose :namespace - expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } + expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count + expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } end class ProjectMember < UserBasic @@ -163,12 +166,15 @@ module API class MergeRequest < ProjectEntity expose :target_branch, :source_branch + # deprecated, always returns 0 + expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels expose :description expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone + expose :merge_when_build_succeeds end class MergeRequestChanges < MergeRequest @@ -192,6 +198,10 @@ module API expose :author, using: Entities::UserBasic expose :created_at expose :system?, as: :system + expose :noteable_id, :noteable_type + # upvote? and downvote? are deprecated, always return false + expose :upvote?, as: :upvote + expose :downvote?, as: :downvote end class MRNote < Grape::Entity @@ -219,6 +229,8 @@ module API expose :target_id, :target_type, :author_id expose :data, :target_title expose :created_at + expose :note, using: Entities::Note, if: ->(event, options) { event.note? } + expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } expose :author_username do |event, options| if event.author @@ -243,7 +255,7 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events + expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -322,7 +334,8 @@ module API end class Release < Grape::Entity - expose :tag, :description + expose :tag, as: :tag_name + expose :description end class RepoTag < Grape::Entity @@ -349,5 +362,9 @@ module API end end end + + class TriggerRequest < Grape::Entity + expose :id, :variables + end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 024aeec2e14..1a14d870a4a 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -65,6 +65,18 @@ module API DestroyGroupService.new(group, current_user).execute end + # Get a list of projects in this group + # + # Example Request: + # GET /groups/:id/projects + get ":id/projects" do + group = find_group(params[:id]) + projects = group.projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::Project + end + # Transfer a project to the Group namespace # # Parameters: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 92540ccf2b1..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -266,12 +266,7 @@ module API projects = projects.search(params[:search]) end - if params[:ci_enabled_first].present? - projects.includes(:gitlab_ci_service). - reorder("services.active DESC, projects.#{project_order_by} #{project_sort}") - else - projects.reorder(project_order_by => project_sort) - end + projects.reorder(project_order_by => project_sort) end def project_order_by diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 6eb84baf9cb..3c1c6bda260 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -76,6 +76,22 @@ module API present merge_request, with: Entities::MergeRequest end + # Show MR commits + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_request/:merge_request_id/commits + # + get ':id/merge_request/:merge_request_id/commits' do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request.commits, with: Entities::RepoCommit + end + # Show MR changes # # Parameters: @@ -179,46 +195,54 @@ module API # Merge MR # # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # merge_commit_message (optional) - Custom merge commit message + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # merge_commit_message (optional) - Custom merge commit message + # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible + # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds # Example: # PUT /projects/:id/merge_request/:merge_request_id/merge # put ":id/merge_request/:merge_request_id/merge" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) - allowed = ::Gitlab::GitAccess.new(current_user, user_project). - can_push_to_branch?(merge_request.target_branch) + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) + not_allowed! if !merge_request.open? || merge_request.work_in_progress? - if allowed - if merge_request.unchecked? - merge_request.check_if_can_be_merged - end + merge_request.check_if_can_be_merged if merge_request.unchecked? - if merge_request.open? && !merge_request.work_in_progress? - if merge_request.can_be_merged? - commit_message = params[:merge_commit_message] || merge_request.merge_commit_message - - ::MergeRequests::MergeService.new(merge_request.target_project, current_user). - execute(merge_request, commit_message) - - present merge_request, with: Entities::MergeRequest - else - render_api_error!('Branch cannot be merged', 405) - end - else - # Merge request can not be merged - # because it is already closed/merged or marked as WIP - not_allowed! - end + render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? + + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? + ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) else - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! + ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) end + + present merge_request, with: Entities::MergeRequest end + # Cancel Merge if Merge When build succeeds is enabled + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # + post ":id/merge_request/:merge_request_id/cancel_merge_when_build_succeeds" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + + ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) + end # Get a merge request's comments # diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 882d1a083ad..cf9938d25a7 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -45,6 +45,7 @@ module API :merge_requests_events, :tag_push_events, :note_events, + :build_events, :enable_ssl_verification ] @hook = user_project.hooks.new(attrs) @@ -77,6 +78,7 @@ module API :merge_requests_events, :tag_push_events, :note_events, + :build_events, :enable_ssl_verification ] diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2b4ada6e2eb..5e75cd35c56 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -7,8 +7,12 @@ module API helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) - publik = parse_boolean(publik) - attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true + if publik.present? && !attrs[:visibility_level].present? + publik = parse_boolean(publik) + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end attrs end end @@ -35,6 +39,17 @@ module API present @projects, with: Entities::Project end + # Gets starred project for the authenticated user + # + # Example Request: + # GET /projects/starred + get '/starred' do + @projects = current_user.starred_projects + @projects = filter_projects(@projects) + @projects = paginate @projects + present @projects, with: Entities::Project + end + # Get all projects for admin user # # Example Request: @@ -78,6 +93,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # shared_runners_enabled (optional) # namespace_id (optional) - defaults to user namespace # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default @@ -94,6 +110,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :shared_runners_enabled, :namespace_id, :public, :visibility_level, @@ -122,6 +139,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) @@ -138,6 +156,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :shared_runners_enabled, :public, :visibility_level, :import_url] @@ -179,6 +198,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project # Example Request @@ -193,6 +213,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :shared_runners_enabled, :public, :visibility_level] attrs = map_public_to_visibility_level(attrs) diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 673342dd447..47621f443e6 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -44,17 +44,42 @@ module API # # Parameters: # id (required) - The ID of a project - # tag (required) - The name of the tag + # tag_name (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # POST /projects/:id/repository/tags/:tag_name/release + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + authorize_push_project + required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + # Updates a release notes of a tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag # description (required) - Release notes with markdown support # Example Request: - # PUT /projects/:id/repository/tags - put ':id/repository/:tag/release', requirements: { tag: /.*/ } do + # PUT /projects/:id/repository/tags/:tag_name/release + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do authorize_push_project required_attributes! [:description] - release = user_project.releases.find_or_initialize_by(tag: params[:tag]) - release.update_attributes(description: params[:description]) + result = UpdateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) - present release, with: Entities::Release + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb new file mode 100644 index 00000000000..2781f1cf191 --- /dev/null +++ b/lib/api/triggers.rb @@ -0,0 +1,48 @@ +module API + # Triggers API + class Triggers < Grape::API + resource :projects do + # Trigger a GitLab project build + # + # Parameters: + # id (required) - The ID of a CI project + # ref (required) - The name of project's branch or tag + # token (required) - The uniq token of trigger + # variables (optional) - The list of variables to be injected into build + # Example Request: + # POST /projects/:id/trigger/builds + post ":id/trigger/builds" do + required_attributes! [:ref, :token] + + project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token].to_s) + not_found! unless project && trigger + unauthorized! unless trigger.project == project + + # validate variables + variables = params[:variables] + if variables + unless variables.is_a?(Hash) + render_api_error!('variables needs to be a hash', 400) + end + + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + # convert variables from Mash to Hash + variables = variables.to_h + end + + # create request and trigger builds + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + if trigger_request + present trigger_request, with: Entities::TriggerRequest + else + errors = 'No builds created' + render_api_error!(errors, 400) + end + end + end + end +end diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index d58a196c4ef..4d99164bc33 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -6,7 +6,42 @@ class AwardEmoji "ambulance", "anguished", "two_hearts", "wink" ] + ALIASES = { + pout: "rage", + satisfied: "laughing", + hankey: "shit", + poop: "shit", + collision: "boom", + thumbsup: "+1", + thumbsdown: "-1", + punch: "facepunch", + raised_hand: "hand", + running: "runner", + ng_woman: "no_good", + shoe: "mans_shoe", + tshirt: "shirt", + honeybee: "bee", + flipper: "dolphin", + paw_prints: "feet", + waxing_gibbous_moon: "moon", + telephone: "phone", + knife: "hocho", + envelope: "email", + pencil: "memo", + open_book: "book", + sailboat: "boat", + red_car: "car", + lantern: "izakaya_lantern", + uk: "gb", + heavy_exclamation_mark: "exclamation", + squirrel: "shipit" + }.with_indifferent_access + def self.path_to_emoji_image(name) "emoji/#{Emoji.emoji_filename(name)}.png" end + + def self.normilize_emoji_name(name) + ALIASES[name] || name + end end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb new file mode 100644 index 00000000000..4153467fbee --- /dev/null +++ b/lib/backup/lfs.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Lfs < Files + def initialize + super('lfs', Settings.lfs.storage_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index e7eda7c6f45..099062eeb8b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -154,7 +154,7 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup diff --git a/lib/banzai.rb b/lib/banzai.rb new file mode 100644 index 00000000000..093382261ae --- /dev/null +++ b/lib/banzai.rb @@ -0,0 +1,13 @@ +module Banzai + def self.render(text, context = {}) + Renderer.render(text, context) + end + + def self.render_result(text, context = {}) + Renderer.render_result(text, context) + end + + def self.post_process(html, context) + Renderer.post_process(html, context) + end +end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb new file mode 100644 index 00000000000..ba2866e1efa --- /dev/null +++ b/lib/banzai/cross_project_reference.rb @@ -0,0 +1,22 @@ +require 'banzai' + +module Banzai + # Common methods for ReferenceFilters that support an optional cross-project + # reference. + module CrossProjectReference + # Given a cross-project reference string, get the Project record + # + # Defaults to value of `context[:project]` if: + # * No reference is given OR + # * Reference given doesn't exist + # + # ref - String reference. + # + # Returns a Project, or nil if the reference can't be found + def project_from_ref(ref) + return context[:project] unless ref + + Project.find_with_namespace(ref) + end + end +end diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb new file mode 100644 index 00000000000..fd4fe024252 --- /dev/null +++ b/lib/banzai/filter.rb @@ -0,0 +1,10 @@ +require 'active_support/core_ext/string/output_safety' +require 'banzai' + +module Banzai + module Filter + def self.[](name) + const_get("#{name.to_s.camelize}Filter") + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb new file mode 100644 index 00000000000..bdaa4721b4b --- /dev/null +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -0,0 +1,145 @@ +require 'banzai' + +module Banzai + module Filter + # Issues, Merge Requests, Snippets, Commits and Commit Ranges share + # similar functionality in reference filtering. + class AbstractReferenceFilter < ReferenceFilter + include CrossProjectReference + + def self.object_class + # Implement in child class + # Example: MergeRequest + end + + def self.object_name + object_class.name.underscore + end + + def self.object_sym + object_name.to_sym + end + + def self.data_reference + "data-#{object_name.dasherize}" + end + + # Public: Find references in text (like `!123` for merge requests) + # + # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| + # object = find_object(project_ref, id) + # "<a href=...>#{object.to_reference}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, an optional String + # of the external project reference, and all of the matchdata. + # + # Returns a String replaced with the return of the block. + def self.references_in(text, pattern = object_class.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[object_sym].to_i, $~[:project], $~ + end + end + + def self.referenced_by(node) + { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } + end + + delegate :object_class, :object_sym, :references_in, to: :class + + def find_object(project, id) + # Implement in child class + # Example: project.merge_requests.find + end + + def url_for_object(object, project) + # Implement in child class + # Example: project_merge_request_url + end + + def call + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end + + # `[Issue](#123)`, which is turned into + # `<a href="#123">Issue</a>` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end + + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end + + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end + end + + # Replace references (like `!123` for merge requests) in text with links + # to the referenced object's details page. + # + # text - String text to replace references in. + # pattern - Reference pattern to match against. + # link_text - Original content of the link being replaced. + # + # Returns a String with references replaced with links. All links + # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. + def object_link_filter(text, pattern, link_text: nil) + references_in(text, pattern) do |match, id, project_ref, matches| + project = project_from_ref(project_ref) + + if project && object = find_object(project, id) + title = escape_once(object_link_title(object)) + klass = reference_class(object_sym) + + data = data_attribute( + original: link_text || match, + project: project.id, + object_sym => object.id + ) + + url = matches[:url] if matches.names.include?("url") + url ||= url_for_object(object, project) + + text = link_text + unless text + text = object.reference_link_text(context[:project]) + + extras = object_link_text_extras(object, matches) + text += " (#{extras.join(", ")})" if extras.any? + end + + %(<a href="#{url}" #{data} + title="#{title}" + class="#{klass}">#{text}</a>) + else + match + end + end + end + + def object_link_text_extras(object, matches) + extras = [] + + if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/ + extras << "comment #{$1}" + end + + extras + end + + def object_link_title(object) + "#{object_class.name.titleize}: #{object.title}" + end + end + end +end diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index c37c3bc55bf..da4ee80c1b5 100644 --- a/lib/gitlab/markdown/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,9 +1,9 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' require 'uri' -module Gitlab - module Markdown +module Banzai + module Filter # HTML Filter for auto-linking URLs in HTML. # # Based on HTML::Pipeline::AutolinkFilter diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb new file mode 100644 index 00000000000..e67cd45ab9b --- /dev/null +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -0,0 +1,58 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces commit range references with links. + # + # This filter supports cross-project references. + class CommitRangeReferenceFilter < AbstractReferenceFilter + def self.object_class + CommitRange + end + + def self.references_in(text, pattern = CommitRange.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit_range], $~[:project], $~ + end + end + + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit-range") + range = find_object(project, id) + + return unless range + + { commit_range: range } + end + + def initialize(*args) + super + + @commit_map = {} + end + + def self.find_object(project, id) + range = CommitRange.new(id, project) + + range.valid_commits? ? range : nil + end + + def find_object(*args) + self.class.find_object(*args) + end + + def url_for_object(range, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_compare_url(project.namespace, project, + range.to_param.merge(only_path: context[:only_path])) + end + + def object_link_title(range) + range.reference_title + end + end + end +end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb new file mode 100644 index 00000000000..9e57608b483 --- /dev/null +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -0,0 +1,63 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces commit references with links. + # + # This filter supports cross-project references. + class CommitReferenceFilter < AbstractReferenceFilter + def self.object_class + Commit + end + + def self.references_in(text, pattern = Commit.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit], $~[:project], $~ + end + end + + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit") + commit = find_object(project, id) + + return unless commit + + { commit: commit } + end + + def self.find_object(project, id) + if project && project.valid_repo? + project.commit(id) + end + end + + def find_object(*args) + self.class.find_object(*args) + end + + def url_for_object(commit, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_commit_url(project.namespace, project, commit, + only_path: context[:only_path]) + end + + def object_link_title(commit) + commit.link_title + end + + def object_link_text_extras(object, matches) + extras = super + + path = matches[:path] if matches.names.include?("path") + if path == '/builds' + extras.unshift "builds" + end + + extras + end + end + end +end diff --git a/lib/gitlab/markdown/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index da10e4d3760..86838e1483c 100644 --- a/lib/gitlab/markdown/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,10 +1,10 @@ require 'action_controller' -require 'gitlab/markdown' +require 'banzai' require 'gitlab_emoji' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces :emoji: with images. # # Based on HTML::Pipeline::EmojiFilter diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 8f86f13976a..f5737a7ac19 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces external issue tracker references with links. # References are ignored if the project doesn't use an external issue # tracker. @@ -30,6 +30,10 @@ module Gitlab replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| issue_link_filter(content) end + + replace_link_nodes_with_href(ExternalIssue.reference_pattern) do |link, text| + issue_link_filter(link, link_text: text) + end end # Replace `JIRA-123` issue references in text with links to the referenced @@ -39,7 +43,7 @@ module Gitlab # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) + def issue_link_filter(text, link_text: nil) project = context[:project] self.class.references_in(text) do |match, issue| @@ -49,9 +53,11 @@ module Gitlab klass = reference_class(:issue) data = data_attribute(project: project.id) + text = link_text || match + %(<a href="#{url}" #{data} title="#{title}" - class="#{klass}">#{match}</a>) + class="#{klass}">#{text}</a>) end end diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 29e51b6ade6..ac87b9820af 100644 --- a/lib/gitlab/markdown/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,16 +1,16 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # HTML Filter to add a `rel="nofollow"` attribute to external links # class ExternalLinkFilter < HTML::Pipeline::Filter def call doc.search('a').each do |node| - next unless node.has_attribute?('href') + link = node.attr('href') - link = node.attribute('href').value + next unless link # Skip non-HTTP(S) links next unless link.start_with?('http') diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 1ed69e2f431..51180cb901a 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces issue references with links. References to # issues that do not exist are ignored. # diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 618acb7a578..07bac2dd7fd 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces label references with links. class LabelReferenceFilter < ReferenceFilter # Public: Find label references in text @@ -30,6 +30,10 @@ module Gitlab replace_text_nodes_matching(Label.reference_pattern) do |content| label_link_filter(content) end + + replace_link_nodes_with_href(Label.reference_pattern) do |link, text| + label_link_filter(link, link_text: text) + end end # Replace label references in text with links to the label specified. @@ -38,7 +42,7 @@ module Gitlab # # Returns a String with label references replaced with links. All links # have `gfm` and `gfm-label` class names attached for styling. - def label_link_filter(text) + def label_link_filter(text, link_text: nil) project = context[:project] self.class.references_in(text) do |match, id, name| @@ -47,10 +51,16 @@ module Gitlab if label = project.labels.find_by(params) url = url_for_label(project, label) klass = reference_class(:label) - data = data_attribute(project: project.id, label: label.id) + data = data_attribute( + original: link_text || match, + project: project.id, + label: label.id + ) + + text = link_text || render_colored_label(label) %(<a href="#{url}" #{data} - class="#{klass}">#{render_colored_label(label)}</a>) + class="#{klass}">#{text}</a>) else match end @@ -59,9 +69,8 @@ module Gitlab def url_for_label(project, label) h = Gitlab::Application.routes.url_helpers - h.namespace_project_issues_path(project.namespace, project, - label_name: label.name, - only_path: context[:only_path]) + h.namespace_project_issues_url( project.namespace, project, label_name: label.name, + only_path: context[:only_path]) end def render_colored_label(label) diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb new file mode 100644 index 00000000000..d09cf41df39 --- /dev/null +++ b/lib/banzai/filter/markdown_filter.rb @@ -0,0 +1,42 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + class MarkdownFilter < HTML::Pipeline::TextFilter + def initialize(text, context = nil, result = nil) + super text, context, result + @text = @text.delete "\r" + end + + def call + html = self.class.renderer.render(@text) + html.rstrip! + html + end + + private + + def self.redcarpet_options + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + @redcarpet_options ||= { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + end + + def self.renderer + @renderer ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + end + end +end diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 1f47f03c94e..755b946a34b 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces merge request references with links. References # to merge requests that do not exist are ignored. # @@ -20,6 +20,22 @@ module Gitlab h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end + + def object_link_text_extras(object, matches) + extras = super + + path = matches[:path] if matches.names.include?("path") + case path + when '/diffs' + extras.unshift "diffs" + when '/commits' + extras.unshift "commits" + when '/builds' + extras.unshift "builds" + end + + extras + end end end end diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index a1f3a8a8ebf..89e7a79789a 100644 --- a/lib/gitlab/markdown/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,8 +1,8 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that removes references to records that the current user does # not have permission to view. # @@ -12,7 +12,10 @@ module Gitlab def call doc.css('a.gfm').each do |node| unless user_can_reference?(node) - node.replace(node.text) + # The reference should be replaced by the original text, + # which is not always the same as the rendered text. + text = node.attr('data-original') || node.text + node.replace(text) end end @@ -24,7 +27,7 @@ module Gitlab def user_can_reference?(node) if node.has_attribute?('data-reference-filter') reference_type = node.attr('data-reference-filter') - reference_filter = reference_type.constantize + reference_filter = Banzai::Filter.const_get(reference_type) reference_filter.user_can_reference?(current_user, node, context) else diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index a4c560f578c..33457a3f361 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,9 +1,9 @@ require 'active_support/core_ext/string/output_safety' -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # Base class for GitLab Flavored Markdown reference filters. # # References within <pre>, <code>, <a>, and <style> elements are ignored. @@ -12,23 +12,6 @@ module Gitlab # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - LazyReference = Struct.new(:klass, :ids) do - def self.load(refs) - lazy_references, values = refs.partition { |ref| ref.is_a?(self) } - - lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| - ids = refs.flat_map(&:ids) - klass.where(id: ids) - end - - values + lazy_values - end - - def load - self.klass.where(id: self.ids) - end - end - def self.user_can_reference?(user, node, context) if node.has_attribute?('data-project') project_id = node.attr('data-project').to_i @@ -53,14 +36,14 @@ module Gitlab # Examples: # # data_attribute(project: 1, issue: 2) - # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" # # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" + # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String def data_attribute(attributes = {}) - attributes[:reference_filter] = self.class.name + attributes[:reference_filter] = self.class.name.demodulize attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") end @@ -122,6 +105,80 @@ module Gitlab doc end + # Iterate through the document's link nodes, yielding the current node's + # content if: + # + # * The `project` context value is present AND + # * The node's content matches `pattern` + # + # pattern - Regex pattern against which to match the node's content + # + # Yields the current node's String contents. The result of the block will + # replace the node and update the current document. + # + # Returns the updated Nokogiri::HTML::DocumentFragment object. + def replace_link_nodes_with_text(pattern) + return doc if project.nil? + + doc.search('a').each do |node| + klass = node.attr('class') + next if klass && klass.include?('gfm') + + link = node.attr('href') + text = node.text + + next unless link && text + + link = URI.decode(link) + # Ignore ending punctionation like periods or commas + next unless link == text && text =~ /\A#{pattern}/ + + html = yield text + + next if html == text + + node.replace(html) + end + + doc + end + + # Iterate through the document's link nodes, yielding the current node's + # content if: + # + # * The `project` context value is present AND + # * The node's HREF matches `pattern` + # + # pattern - Regex pattern against which to match the node's HREF + # + # Yields the current node's String HREF and String content. + # The result of the block will replace the node and update the current document. + # + # Returns the updated Nokogiri::HTML::DocumentFragment object. + def replace_link_nodes_with_href(pattern) + return doc if project.nil? + + doc.search('a').each do |node| + klass = node.attr('class') + next if klass && klass.include?('gfm') + + link = node.attr('href') + text = node.text + + next unless link && text + link = URI.decode(link) + next unless link && link =~ /\A#{pattern}\z/ + + html = yield link, text + + next if html == link + + node.replace(html) + end + + doc + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 00f983675e6..855f238ac1e 100644 --- a/lib/gitlab/markdown/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,8 +1,8 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that gathers all referenced records that the current user has # permission to view. # @@ -20,7 +20,7 @@ module Gitlab gather_references(node) end - load_lazy_references unless context[:load_lazy_references] == false + load_lazy_references unless ReferenceExtractor.lazy? doc end @@ -31,7 +31,7 @@ module Gitlab return unless node.has_attribute?('data-reference-filter') reference_type = node.attr('data-reference-filter') - reference_filter = reference_type.constantize + reference_filter = Banzai::Filter.const_get(reference_type) return if context[:reference_filter] && reference_filter != context[:reference_filter] @@ -47,11 +47,10 @@ module Gitlab end end - # Will load all references of one type using one query. def load_lazy_references refs = result[:references] refs.each do |type, values| - refs[type] = ReferenceFilter::LazyReference.load(values) + refs[type] = ReferenceExtractor.lazily(values) end end diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 632be4d7542..5a081125f21 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,9 +1,9 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' require 'uri' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that "fixes" relative links to files in a repository. # # Context options: @@ -16,7 +16,7 @@ module Gitlab def call return doc unless linkable_files? - doc.search('a').each do |el| + doc.search('a:not(.gfm)').each do |el| process_link_attr el.attribute('href') end diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index ffb9dc33b64..d03e3ae4b3c 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,9 +1,9 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' -module Gitlab - module Markdown +module Banzai + module Filter # Sanitize HTML # # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. @@ -11,7 +11,7 @@ module Gitlab def whitelist # Descriptions are more heavily sanitized, allowing only a few elements. # See http://git.io/vkuAN - if pipeline == :description + if context[:inline_sanitization] whitelist = LIMITED whitelist[:elements] -= %w(pre code img ol ul li) else @@ -25,10 +25,6 @@ module Gitlab private - def pipeline - context[:pipeline] || :default - end - def customized?(transformers) transformers.last.source_location[0] == __FILE__ end diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index f7bd07c2a34..1ad5df96f85 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces snippet references with links. References to # snippets that do not exist are ignored. # diff --git a/lib/gitlab/markdown/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 8597e02f0de..c889cc1e97c 100644 --- a/lib/gitlab/markdown/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,9 +1,9 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' -module Gitlab - module Markdown +module Banzai + module Filter # HTML Filter to highlight fenced code blocks # class SyntaxHighlightFilter < HTML::Pipeline::Filter diff --git a/lib/gitlab/markdown/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index bbb3bf7fc8b..9b3e67206d5 100644 --- a/lib/gitlab/markdown/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,8 +1,8 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that adds an anchor child element to all Headers in a # document, so that they can be linked to. # @@ -31,7 +31,7 @@ module Gitlab id = text.downcase id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.gsub!(' ', '-') # replace spaces with dash + id.tr!(' ', '-') # replace spaces with dash id.squeeze!('-') # replace multiple dashes with one uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' diff --git a/lib/gitlab/markdown/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 2f133ae8500..bdf7c2ebdfc 100644 --- a/lib/gitlab/markdown/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,8 +1,8 @@ -require 'gitlab/markdown' +require 'banzai' require 'task_list/filter' -module Gitlab - module Markdown +module Banzai + module Filter # Work around a bug in the default TaskList::Filter that adds a `task-list` # class to every list element, regardless of whether or not it contains a # task list. diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index fbada73ab86..1a1d0aad8ca 100644 --- a/lib/gitlab/markdown/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,9 +1,9 @@ -require 'gitlab/markdown' +require 'banzai' require 'html/pipeline/filter' require 'uri' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that "fixes" relative upload links to files. # Context options: # :project (required) - Current project diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index ab5e1f6fe9e..67c24faf991 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,7 +1,7 @@ -require 'gitlab/markdown' +require 'banzai' -module Gitlab - module Markdown +module Banzai + module Filter # HTML filter that replaces user or group references with links. # # A special `@all` reference is also supported. @@ -52,6 +52,10 @@ module Gitlab replace_text_nodes_matching(User.reference_pattern) do |content| user_link_filter(content) end + + replace_link_nodes_with_href(User.reference_pattern) do |link, text| + user_link_filter(link, link_text: text) + end end # Replace `@user` user references in text with links to the referenced @@ -61,12 +65,12 @@ module Gitlab # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text) + def user_link_filter(text, link_text: nil) self.class.references_in(text) do |match, username| if username == 'all' - link_to_all + link_to_all(link_text: link_text) elsif namespace = Namespace.find_by(path: username) - link_to_namespace(namespace) || match + link_to_namespace(namespace, link_text: link_text) || match else match end @@ -83,36 +87,36 @@ module Gitlab reference_class(:project_member) end - def link_to_all + def link_to_all(link_text: nil) project = context[:project] url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) data = data_attribute(project: project.id) - text = User.reference_prefix + 'all' + text = link_text || User.reference_prefix + 'all' link_tag(url, data, text) end - def link_to_namespace(namespace) + def link_to_namespace(namespace, link_text: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace) + link_to_group(namespace.path, namespace, link_text: link_text) else - link_to_user(namespace.path, namespace) + link_to_user(namespace.path, namespace, link_text: link_text) end end - def link_to_group(group, namespace) + def link_to_group(group, namespace, link_text: nil) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = Group.reference_prefix + group + text = link_text || Group.reference_prefix + group link_tag(url, data, text) end - def link_to_user(user, namespace) + def link_to_user(user, namespace, link_text: nil) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = User.reference_prefix + user + text = link_text || User.reference_prefix + user link_tag(url, data, text) end diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb new file mode 100644 index 00000000000..073ec5d9801 --- /dev/null +++ b/lib/banzai/lazy_reference.rb @@ -0,0 +1,27 @@ +require 'banzai' + +module Banzai + class LazyReference + def self.load(refs) + lazy_references, values = refs.partition { |ref| ref.is_a?(self) } + + lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| + ids = refs.flat_map(&:ids) + klass.where(id: ids) + end + + values + lazy_values + end + + attr_reader :klass, :ids + + def initialize(klass, ids) + @klass = klass + @ids = Array.wrap(ids).map(&:to_i) + end + + def load + self.klass.where(id: self.ids) + end + end +end diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb new file mode 100644 index 00000000000..4e017809d9d --- /dev/null +++ b/lib/banzai/pipeline.rb @@ -0,0 +1,10 @@ +require 'banzai' + +module Banzai + module Pipeline + def self.[](name) + name ||= :full + const_get("#{name.to_s.camelize}Pipeline") + end + end +end diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb new file mode 100644 index 00000000000..5e76a817be5 --- /dev/null +++ b/lib/banzai/pipeline/asciidoc_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class AsciidocPipeline < BasePipeline + def self.filters + [ + Filter::RelativeLinkFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb new file mode 100644 index 00000000000..957f352aec5 --- /dev/null +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -0,0 +1,14 @@ +require 'banzai' + +module Banzai + module Pipeline + class AtomPipeline < FullPipeline + def self.transform_context(context) + super(context).merge( + only_path: false, + xhtml: true + ) + end + end + end +end diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb new file mode 100644 index 00000000000..cd30009e5c0 --- /dev/null +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -0,0 +1,30 @@ +require 'banzai' +require 'html/pipeline' + +module Banzai + module Pipeline + class BasePipeline + def self.filters + [] + end + + def self.transform_context(context) + context + end + + def self.html_pipeline + @html_pipeline ||= HTML::Pipeline.new(filters) + end + + class << self + %i(call to_document to_html).each do |meth| + define_method(meth) do |text, context| + context = transform_context(context) + + html_pipeline.send(meth, text, context) + end + end + end + end + end +end diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb new file mode 100644 index 00000000000..f3bf1809d18 --- /dev/null +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -0,0 +1,27 @@ +require 'banzai' + +module Banzai + module Pipeline + module CombinedPipeline + def self.new(*pipelines) + Class.new(BasePipeline) do + const_set :PIPELINES, pipelines + + def self.pipelines + self::PIPELINES + end + + def self.filters + pipelines.flat_map(&:filters) + end + + def self.transform_context(context) + pipelines.reduce(context) do |context, pipeline| + pipeline.transform_context(context) + end + end + end + end + end + end +end diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb new file mode 100644 index 00000000000..94c2cb165a5 --- /dev/null +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -0,0 +1,14 @@ +require 'banzai' + +module Banzai + module Pipeline + class DescriptionPipeline < FullPipeline + def self.transform_context(context) + super(context).merge( + # SanitizationFilter + inline_sanitization: true + ) + end + end + end +end diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb new file mode 100644 index 00000000000..14356145a35 --- /dev/null +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class EmailPipeline < FullPipeline + def self.transform_context(context) + super(context).merge( + only_path: false + ) + end + end + end +end diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb new file mode 100644 index 00000000000..72395a5d50e --- /dev/null +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -0,0 +1,9 @@ +require 'banzai' + +module Banzai + module Pipeline + class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) + + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb new file mode 100644 index 00000000000..38750b55ec7 --- /dev/null +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -0,0 +1,41 @@ +require 'banzai' + +module Banzai + module Pipeline + class GfmPipeline < BasePipeline + def self.filters + @filters ||= [ + Filter::SyntaxHighlightFilter, + Filter::SanitizationFilter, + + Filter::UploadLinkFilter, + Filter::EmojiFilter, + Filter::TableOfContentsFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter, + + Filter::UserReferenceFilter, + Filter::IssueReferenceFilter, + Filter::ExternalIssueReferenceFilter, + Filter::MergeRequestReferenceFilter, + Filter::SnippetReferenceFilter, + Filter::CommitRangeReferenceFilter, + Filter::CommitReferenceFilter, + Filter::LabelReferenceFilter, + + Filter::TaskListFilter + ] + end + + def self.transform_context(context) + context.merge( + only_path: true, + + # EmojiFilter + asset_host: Gitlab::Application.config.asset_host, + asset_root: Gitlab.config.gitlab.base_url + ) + end + end + end +end diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb new file mode 100644 index 00000000000..89335143852 --- /dev/null +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -0,0 +1,14 @@ +require 'banzai' + +module Banzai + module Pipeline + class NotePipeline < FullPipeline + def self.transform_context(context) + super(context).merge( + # TableOfContentsFilter + no_header_anchors: true + ) + end + end + end +end diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb new file mode 100644 index 00000000000..998fd75daa2 --- /dev/null +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class PlainMarkdownPipeline < BasePipeline + def self.filters + [ + Filter::MarkdownFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb new file mode 100644 index 00000000000..148f24b6ce1 --- /dev/null +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -0,0 +1,20 @@ +require 'banzai' + +module Banzai + module Pipeline + class PostProcessPipeline < BasePipeline + def self.filters + [ + Filter::RelativeLinkFilter, + Filter::RedactorFilter + ] + end + + def self.transform_context(context) + context.merge( + post_process: true + ) + end + end + end +end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb new file mode 100644 index 00000000000..4f9bc9fcccc --- /dev/null +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class ReferenceExtractionPipeline < BasePipeline + def self.filters + [ + Filter::ReferenceGathererFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb new file mode 100644 index 00000000000..6725c9039a9 --- /dev/null +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -0,0 +1,9 @@ +require 'banzai' + +module Banzai + module Pipeline + class SingleLinePipeline < GfmPipeline + + end + end +end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb new file mode 100644 index 00000000000..2c197d31898 --- /dev/null +++ b/lib/banzai/reference_extractor.rb @@ -0,0 +1,55 @@ +require 'banzai' + +module Banzai + # Extract possible GFM references from an arbitrary String for further processing. + class ReferenceExtractor + class << self + LAZY_KEY = :banzai_reference_extractor_lazy + + def lazy? + Thread.current[LAZY_KEY] + end + + def lazily(values = nil, &block) + return (values || block.call).uniq if lazy? + + begin + Thread.current[LAZY_KEY] = true + + values ||= block.call + + Banzai::LazyReference.load(values.uniq).uniq + ensure + Thread.current[LAZY_KEY] = false + end + end + end + + def initialize + @texts = [] + end + + def analyze(text, context = {}) + @texts << Renderer.render(text, context) + end + + def references(type, context = {}) + filter = Banzai::Filter["#{type}_reference"] + + context.merge!( + pipeline: :reference_extraction, + + # ReferenceGathererFilter + reference_filter: filter + ) + + self.class.lazily do + @texts.flat_map do |html| + text_context = context.dup + result = Renderer.render_result(html, text_context) + result[:references][type] + end.uniq + end + end + end +end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb new file mode 100644 index 00000000000..115ae914524 --- /dev/null +++ b/lib/banzai/renderer.rb @@ -0,0 +1,78 @@ +module Banzai + module Renderer + CACHE_ENABLED = false + + # Convert a Markdown String into an HTML-safe String of HTML + # + # Note that while the returned HTML will have been sanitized of dangerous + # HTML, it may post a risk of information leakage if it's not also passed + # through `post_process`. + # + # Also note that the returned String is always HTML, not XHTML. Views + # requiring XHTML, such as Atom feeds, need to call `post_process` on the + # result, providing the appropriate `pipeline` option. + # + # markdown - Markdown String + # context - Hash of context options passed to our HTML Pipeline + # + # Returns an HTML-safe String + def self.render(text, context = {}) + cache_key = context.delete(:cache_key) + cache_key = full_cache_key(cache_key, context[:pipeline]) + + if cache_key && CACHE_ENABLED + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) + end + else + cacheless_render(text, context) + end + end + + def self.render_result(text, context = {}) + Pipeline[context[:pipeline]].call(text, context) + end + + # Perform post-processing on an HTML String + # + # This method is used to perform state-dependent changes to a String of + # HTML, such as removing references that the current user doesn't have + # permission to make (`RedactorFilter`). + # + # html - String to process + # context - Hash of options to customize output + # :pipeline - Symbol pipeline type + # :project - Project + # :user - User object + # + # Returns an HTML-safe String + def self.post_process(html, context) + context = Pipeline[context[:pipeline]].transform_context(context) + + pipeline = Pipeline[:post_process] + if context[:xhtml] + pipeline.to_document(html, context).to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) + else + pipeline.to_html(html, context) + end.html_safe + end + + private + + def self.cacheless_render(text, context = {}) + result = render_result(text, context) + + output = result[:output] + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end + end + + def self.full_cache_key(cache_key, pipeline_name) + return unless cache_key + ["banzai", *cache_key, pipeline_name || :full] + end + end +end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 07e68216d7f..5c347e432b4 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -30,9 +30,7 @@ module Ci helpers Gitlab::CurrentSettings mount Builds - mount Commits mount Runners - mount Projects mount Triggers end end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 0a586672807..15faa6edd84 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -58,6 +58,7 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) @@ -91,6 +92,7 @@ module Ci # POST /builds/:id/artifacts post ":id/artifacts" do require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb deleted file mode 100644 index a60769d8305..00000000000 --- a/lib/ci/api/commits.rb +++ /dev/null @@ -1,66 +0,0 @@ -module Ci - module API - class Commits < Grape::API - resource :commits do - # Get list of commits per project - # - # Parameters: - # project_id (required) - The ID of a project - # project_token (requires) - Project token - # page (optional) - # per_page (optional) - items per request (default is 20) - # - get do - required_attributes! [:project_id, :project_token] - project = Ci::Project.find(params[:project_id]) - authenticate_project_token!(project) - - commits = project.commits.page(params[:page]).per(params[:per_page] || 20) - present commits, with: Entities::CommitWithBuilds - end - - # Create a commit - # - # Parameters: - # project_id (required) - The ID of a project - # project_token (requires) - Project token - # data (required) - GitLab push data - # - # Sample GitLab push data: - # { - # "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", - # "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", - # "ref": "refs/heads/master", - # "commits": [ - # { - # "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", - # "message": "Update Catalan translation to e38cb41.", - # "timestamp": "2011-12-12T14:27:31+02:00", - # "url": "http://localhost/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", - # "author": { - # "name": "Jordi Mallach", - # "email": "jordi@softcatala.org", - # } - # }, .... more commits - # ] - # } - # - # Example Request: - # POST /commits - post do - required_attributes! [:project_id, :data, :project_token] - project = Ci::Project.find(params[:project_id]) - authenticate_project_token!(project) - commit = Ci::CreateCommitService.new.execute(project, current_user, params[:data]) - - if commit.persisted? - present commit, with: Entities::CommitWithBuilds - else - errors = commit.errors.full_messages.join(", ") - render_api_error!(errors, 400) - end - end - end - end - end -end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 750f421872d..e4ac0545ea2 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -37,15 +37,6 @@ module Ci expose :id, :token end - class Project < Grape::Entity - expose :id, :name, :token, :default_ref, :gitlab_url, :path, - :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id - - expose :timeout do |model| - model.timeout - end - end - class RunnerProject < Grape::Entity expose :id, :project_id, :runner_id end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 02502333756..443563c2e4a 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -6,22 +6,22 @@ module Ci UPDATE_RUNNER_EVERY = 60 def authenticate_runners! - forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN + forbidden! unless runner_registration_token_valid? end def authenticate_runner! forbidden! unless current_runner end - def authenticate_project_token!(project) - forbidden! unless project.valid_token?(params[:project_token]) - end - def authenticate_build_token!(build) token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s forbidden! unless token && build.valid_token?(token) end + def runner_registration_token_valid? + params[:token] == current_application_settings.ensure_runners_registration_token + end + def update_runner_last_contact # Use a random threshold to prevent beating DB updates contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb deleted file mode 100644 index d719ad9e8d5..00000000000 --- a/lib/ci/api/projects.rb +++ /dev/null @@ -1,195 +0,0 @@ -module Ci - module API - # Projects API - class Projects < Grape::API - before { authenticate! } - - resource :projects do - # Register new webhook for project - # - # Parameters - # project_id (required) - The ID of a project - # web_hook (required) - WebHook URL - # Example Request - # POST /projects/:project_id/webhooks - post ":project_id/webhooks" do - required_attributes! [:web_hook] - - project = Ci::Project.find(params[:project_id]) - - unauthorized! unless can?(current_user, :admin_project, project.gl_project) - - web_hook = project.web_hooks.new({ url: params[:web_hook] }) - - if web_hook.save - present web_hook, with: Entities::WebHook - else - errors = web_hook.errors.full_messages.join(", ") - render_api_error!(errors, 400) - end - end - - # Retrieve all Gitlab CI projects that the user has access to - # - # Example Request: - # GET /projects - get do - gitlab_projects = current_user.authorized_projects - gitlab_projects = filter_projects(gitlab_projects) - gitlab_projects = paginate gitlab_projects - - ids = gitlab_projects.map { |project| project.id } - - projects = Ci::Project.where("gitlab_id IN (?)", ids).load - present projects, with: Entities::Project - end - - # Retrieve all Gitlab CI projects that the user owns - # - # Example Request: - # GET /projects/owned - get "owned" do - gitlab_projects = current_user.owned_projects - gitlab_projects = filter_projects(gitlab_projects) - gitlab_projects = paginate gitlab_projects - - ids = gitlab_projects.map { |project| project.id } - - projects = Ci::Project.where("gitlab_id IN (?)", ids).load - present projects, with: Entities::Project - end - - # Retrieve info for a Gitlab CI project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id - get ":id" do - project = Ci::Project.find(params[:id]) - unauthorized! unless can?(current_user, :read_project, project.gl_project) - - present project, with: Entities::Project - end - - # Create Gitlab CI project using Gitlab project info - # - # Parameters: - # gitlab_id (required) - The gitlab id of the project - # default_ref - The branch to run against (defaults to `master`) - # Example Request: - # POST /projects - post do - required_attributes! [:gitlab_id] - - filtered_params = { - gitlab_id: params[:gitlab_id], - # we accept gitlab_url for backward compatibility for a while (added to 7.11) - default_ref: params[:default_ref] || 'master' - } - - project = Ci::Project.new(filtered_params) - project.build_missing_services - - if project.save - present project, with: Entities::Project - else - errors = project.errors.full_messages.join(", ") - render_api_error!(errors, 400) - end - end - - # Update a Gitlab CI project - # - # Parameters: - # id (required) - The ID of a project - # default_ref - The branch to run against (defaults to `master`) - # Example Request: - # PUT /projects/:id - put ":id" do - project = Ci::Project.find(params[:id]) - - unauthorized! unless can?(current_user, :admin_project, project.gl_project) - - attrs = attributes_for_keys [:default_ref] - - if project.update_attributes(attrs) - present project, with: Entities::Project - else - errors = project.errors.full_messages.join(", ") - render_api_error!(errors, 400) - end - end - - # Remove a Gitlab CI project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id - delete ":id" do - project = Ci::Project.find(params[:id]) - - unauthorized! unless can?(current_user, :admin_project, project.gl_project) - - project.destroy - end - - # Link a Gitlab CI project to a runner - # - # Parameters: - # id (required) - The ID of a CI project - # runner_id (required) - The ID of a runner - # Example Request: - # POST /projects/:id/runners/:runner_id - post ":id/runners/:runner_id" do - project = Ci::Project.find(params[:id]) - runner = Ci::Runner.find(params[:runner_id]) - - unauthorized! unless can?(current_user, :admin_project, project.gl_project) - - options = { - project_id: project.id, - runner_id: runner.id - } - - runner_project = Ci::RunnerProject.new(options) - - if runner_project.save - present runner_project, with: Entities::RunnerProject - else - errors = project.errors.full_messages.join(", ") - render_api_error!(errors, 400) - end - end - - # Remove a Gitlab CI project from a runner - # - # Parameters: - # id (required) - The ID of a CI project - # runner_id (required) - The ID of a runner - # Example Request: - # DELETE /projects/:id/runners/:runner_id - delete ":id/runners/:runner_id" do - project = Ci::Project.find(params[:id]) - runner = Ci::Runner.find(params[:runner_id]) - - unauthorized! unless can?(current_user, :admin_project, project.gl_project) - - options = { - project_id: project.id, - runner_id: runner.id - } - - runner_project = Ci::RunnerProject.find_by(options) - - if runner_project.present? - runner_project.destroy - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 1466fe4356e..bfc14fe7a6b 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -3,17 +3,6 @@ module Ci # Runners API class Runners < Grape::API resource :runners do - # Get list of all available runners - # - # Example Request: - # GET /runners - get do - authenticate! - runners = Ci::Runner.all - - present runners, with: Entities::Runner - end - # Delete runner # Parameters: # token (required) - The unique token of runner @@ -40,14 +29,14 @@ module Ci required_attributes! [:token] runner = - if params[:token] == GitlabCi::REGISTRATION_TOKEN + if runner_registration_token_valid? # Create shared runner. Requires admin access Ci::Runner.create( description: params[:description], tag_list: params[:tag_list], is_shared: true ) - elsif project = Ci::Project.find_by(token: params[:token]) + elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. project.runners.create( description: params[:description], diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb index 40907d6db54..63b42113513 100644 --- a/lib/ci/api/triggers.rb +++ b/lib/ci/api/triggers.rb @@ -14,7 +14,7 @@ module Ci post ":id/refs/:ref/trigger" do required_attributes! [:token] - project = Ci::Project.find(params[:id]) + project = Project.find_by(ci_id: params[:id].to_i) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger unauthorized! unless trigger.project == project diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 5ff7407c6fe..d53bdcbd0f2 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,7 +60,7 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.last(30) + commits = project.ci_commits.last(30) commits.each do |commit| @labels << commit.short_sha diff --git a/lib/ci/current_settings.rb b/lib/ci/current_settings.rb deleted file mode 100644 index fd78b024970..00000000000 --- a/lib/ci/current_settings.rb +++ /dev/null @@ -1,22 +0,0 @@ -module Ci - module CurrentSettings - def current_application_settings - key = :ci_current_application_settings - - RequestStore.store[key] ||= begin - if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('ci_application_settings') - Ci::ApplicationSetting.current || Ci::ApplicationSetting.create_from_defaults - else - fake_application_settings - end - end - end - - def fake_application_settings - OpenStruct.new( - all_broken_builds: Ci::Settings.gitlab_ci['all_broken_builds'], - add_pusher: Ci::Settings.gitlab_ci['add_pusher'], - ) - end - end -end diff --git a/lib/ci/git.rb b/lib/ci/git.rb deleted file mode 100644 index 7acc3f38edb..00000000000 --- a/lib/ci/git.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Ci - module Git - BLANK_SHA = '0' * 40 - end -end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3beafcad117..bcdfd38d292 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -10,7 +10,7 @@ module Ci attr_reader :before_script, :image, :services, :variables, :path, :cache def initialize(config, path = nil) - @config = YAML.load(config) + @config = YAML.safe_load(config, [Symbol]) @path = path unless @config.is_a? Hash @@ -132,26 +132,36 @@ module Ci end def validate_job!(name, job) + validate_job_name!(name) + validate_job_keys!(name, job) + validate_job_types!(name, job) + + validate_job_stage!(name, job) if job[:stage] + validate_job_cache!(name, job) if job[:cache] + validate_job_artifacts!(name, job) if job[:artifacts] + end + + private + + def validate_job_name!(name) if name.blank? || !validate_string(name) raise ValidationError, "job name should be non-empty string" end + end + def validate_job_keys!(name, job) job.keys.each do |key| unless ALLOWED_JOB_KEYS.include? key raise ValidationError, "#{name} job: unknown parameter #{key}" end end + end + def validate_job_types!(name, job) if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) raise ValidationError, "#{name} job: script should be a string or an array of a strings" end - if job[:stage] - unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" - end - end - if job[:image] && !validate_string(job[:image]) raise ValidationError, "#{name} job: image should be a string" end @@ -172,36 +182,40 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end - if job[:cache] - if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) - raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" - end - - if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) - raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" - end + if job[:allow_failure] && !validate_boolean(job[:allow_failure]) + raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:artifacts] - if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) - raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" - end + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" + end + end - if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) - raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" - end + def validate_job_stage!(name, job) + unless job[:stage].is_a?(String) && job[:stage].in?(stages) + raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" end + end - if job[:allow_failure] && !validate_boolean(job[:allow_failure]) - raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" + def validate_job_cache!(name, job) + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) + raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w(on_success on_failure always)) - raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" + if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) + raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" end end - private + def validate_job_artifacts!(name, job) + if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) + raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" + end + + if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) + raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" + end + end def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } diff --git a/lib/ci/scheduler.rb b/lib/ci/scheduler.rb deleted file mode 100644 index ee0958f4be1..00000000000 --- a/lib/ci/scheduler.rb +++ /dev/null @@ -1,16 +0,0 @@ -module Ci - class Scheduler - def perform - projects = Ci::Project.where(always_build: true).all - projects.each do |project| - last_commit = project.commits.last - next unless last_commit && last_commit.last_build - - interval = project.polling_interval - if (last_commit.last_build.created_at + interval.hours) < Time.now - last_commit.retry - end - end - end - end -end diff --git a/lib/email_validator.rb b/lib/email_validator.rb deleted file mode 100644 index f509f0a5843..00000000000 --- a/lib/email_validator.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Based on https://github.com/balexand/email_validator -# -# Extended to use only strict mode with following allowed characters: -# ' - apostrophe -# -# See http://www.remote.org/jochen/mail/info/chars.html -# -class EmailValidator < ActiveModel::EachValidator - @@default_options = {} - - def self.default_options - @@default_options - end - - def validate_each(record, attribute, value) - options = @@default_options.merge(self.options) - unless value =~ /\A\s*([-a-z0-9+._']{1,64})@((?:[-a-z0-9]+\.)+[a-z]{2,})\s*\z/i - record.errors.add(attribute, options[:message] || :invalid) - end - end -end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index bf33e5b1b1e..b203b9d70e4 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,14 +1,10 @@ require 'asciidoctor' -require 'html/pipeline' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - # Provide autoload paths for filters to prevent a circular dependency error - autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' @@ -24,13 +20,11 @@ module Gitlab # :requested_path # :ref # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter - # html_opts - a Hash of options for HTML output: - # :xhtml - output XHTML instead of HTML # - def self.render(input, context, asciidoc_opts = {}, html_opts = {}) - asciidoc_opts = asciidoc_opts.reverse_merge( + def self.render(input, context, asciidoc_opts = {}) + asciidoc_opts.reverse_merge!( safe: :secure, - backend: html_opts[:xhtml] ? :xhtml5 : :html5, + backend: :html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) @@ -38,23 +32,10 @@ module Gitlab html = ::Asciidoctor.convert(input, asciidoc_opts) if context[:project] - result = HTML::Pipeline.new(filters).call(html, context) - - save_opts = html_opts[:xhtml] ? - Nokogiri::XML::Node::SaveOptions::AS_XHTML : 0 - - html = result[:output].to_html(save_with: save_opts) + html = Banzai.render(html, context.merge(pipeline: :asciidoc)) end html.html_safe end - - private - - def self.filters - [ - Gitlab::Markdown::RelativeLinkFilter - ] - end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 0d156047ff0..cdcaae8094c 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -77,7 +77,9 @@ module Grack if project && matched_login.present? && git_cmd == 'git-upload-pack' underscored_service = matched_login['s'].underscore - if Service.available_services_names.include?(underscored_service) + if underscored_service == 'gitlab_ci' + return project && project.valid_build_token?(password) + elsif Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" service = project.send(service_method) diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 87ac30b5ffe..459e3d6bcdb 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -2,7 +2,7 @@ module Gitlab class Shell class Error < StandardError; end - class KeyAdder < Struct.new(:io) + KeyAdder = Struct.new(:io) do def add_key(id, key) key.gsub!(/[[:space:]]+/, ' ').strip! io.puts("#{id}\t#{key}") diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 35e34d033e0..03aac1a025a 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["slug"], description: repo["description"], diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb deleted file mode 100644 index 43145e0ee1b..00000000000 --- a/lib/gitlab/blacklist.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Blacklist - extend self - - def path - %w( - admin - dashboard - files - groups - help - profile - projects - search - public - assets - u - s - teams - merge_requests - issues - users - snippets - services - repository - hooks - notes - unsubscribes - all - ci - ) - end - end -end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb new file mode 100644 index 00000000000..86bfa0a4378 --- /dev/null +++ b/lib/gitlab/build_data_builder.rb @@ -0,0 +1,64 @@ +module Gitlab + class BuildDataBuilder + class << self + def build(build) + project = build.project + commit = build.commit + user = build.user + + data = { + object_kind: 'build', + + ref: build.ref, + tag: build.tag, + before_sha: build.before_sha, + sha: build.sha, + + # TODO: should this be not prefixed with build_? + # Leaving this way to have backward compatibility + build_id: build.id, + build_name: build.name, + build_stage: build.stage, + build_status: build.status, + build_started_at: build.started_at, + build_finished_at: build.finished_at, + build_duration: build.duration, + + # TODO: do we still need it? + project_id: project.id, + project_name: project.name_with_namespace, + + user: { + id: user.try(:id), + name: user.try(:name), + email: user.try(:email), + }, + + commit: { + id: commit.id, + sha: commit.sha, + message: commit.git_commit_message, + author_name: commit.git_author_name, + author_email: commit.git_author_email, + status: commit.status, + duration: commit.duration, + started_at: commit.started_at, + finished_at: commit.finished_at, + }, + + repository: { + name: project.name, + url: project.url_to_repo, + description: project.description, + homepage: project.web_url, + git_http_url: project.http_url_to_repo, + git_ssh_url: project.ssh_url_to_repo, + visibility_level: project.visibility_level + }, + } + + data + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index aeec595782c..9bef9037ad6 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,6 +1,12 @@ module Gitlab class ClosingIssueExtractor - ISSUE_CLOSING_REGEX = Regexp.new(Gitlab.config.gitlab.issue_closing_pattern) + ISSUE_CLOSING_REGEX = begin + link_pattern = URI.regexp(%w(http https)) + + pattern = Gitlab.config.gitlab.issue_closing_pattern + pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))") + Regexp.new(pattern).freeze + end def initialize(project, current_user = nil) @extractor = Gitlab::ReferenceExtractor.new(project, current_user) @@ -9,10 +15,12 @@ module Gitlab def closed_by_message(message) return [] if message.nil? - closing_statements = message.scan(ISSUE_CLOSING_REGEX). - map { |ref| ref[0] }.join(" ") + closing_statements = [] + message.scan(ISSUE_CLOSING_REGEX) do + closing_statements << Regexp.last_match[0] + end - @extractor.analyze(closing_statements) + @extractor.analyze(closing_statements.join(" ")) @extractor.issues end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2d3e32d9539..46a4ef0e31f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab session_expire_delay: Settings.gitlab['session_expire_delay'], import_sources: Settings.gitlab['import_sources'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], + max_artifacts_size: Settings.artifacts['max_size'], ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 71f37f1fef8..de77a6fbff1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -7,5 +7,23 @@ module Gitlab def self.postgresql? ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' end + + def true_value + case ActiveRecord::Base.connection.adapter_name.downcase + when 'postgresql' + "'t'" + else + 1 + end + end + + def false_value + case ActiveRecord::Base.connection.adapter_name.downcase + when 'postgresql' + "'f'" + else + 0 + end + end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 142058aa69d..79061cd0141 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -46,11 +46,11 @@ module Gitlab end def added_lines - diff_lines.select(&:added?).size + diff_lines.count(&:added?) end def removed_lines - diff_lines.select(&:removed?).size + diff_lines.count(&:removed?) end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb new file mode 100644 index 00000000000..a2eb7a70bd2 --- /dev/null +++ b/lib/gitlab/email/message/repository_push.rb @@ -0,0 +1,137 @@ +module Gitlab + module Email + module Message + class RepositoryPush + attr_accessor :recipient + attr_reader :author_id, :ref, :action + + include Gitlab::Application.routes.url_helpers + + delegate :namespace, :name_with_namespace, to: :project, prefix: :project + delegate :name, to: :author, prefix: :author + + def initialize(notify, project_id, recipient, opts = {}) + raise ArgumentError, 'Missing options: author_id, ref, action' unless + opts[:author_id] && opts[:ref] && opts[:action] + + @notify = notify + @project_id = project_id + @recipient = recipient + @opts = opts.dup + + @author_id = @opts.delete(:author_id) + @ref = @opts.delete(:ref) + @action = @opts.delete(:action) + end + + def project + @project ||= Project.find(@project_id) + end + + def author + @author ||= User.find(@author_id) + end + + def commits + @commits ||= (Commit.decorate(compare.commits, project) if compare) + end + + def diffs + @diffs ||= (compare.diffs if compare) + end + + def diffs_count + diffs.count if diffs + end + + def compare + @opts[:compare] + end + + def compare_timeout + compare.timeout if compare + end + + def reverse_compare? + @opts[:reverse_compare] || false + end + + def disable_diffs? + @opts[:disable_diffs] || false + end + + def send_from_committer_email? + @opts[:send_from_committer_email] || false + end + + def action_name + @action_name ||= + case @action + when :create + 'pushed new' + when :delete + 'deleted' + else + 'pushed to' + end + end + + def ref_name + @ref_name ||= Gitlab::Git.ref_name(@ref) + end + + def ref_type + @ref_type ||= Gitlab::Git.tag_ref?(@ref) ? 'tag' : 'branch' + end + + def target_url + if @action == :push && commits + if commits.length > 1 + namespace_project_compare_url(project_namespace, + project, + from: Commit.new(compare.base, project), + to: Commit.new(compare.head, project)) + else + namespace_project_commit_url(project_namespace, + project, commits.first) + end + else + unless @action == :delete + namespace_project_tree_url(project_namespace, + project, ref_name) + end + end + end + + def reply_to + if send_from_committer_email? && @notify.can_send_from_user_email?(author) + author.email + else + Gitlab.config.gitlab.email_reply_to + end + end + + def subject + subject_text = '[Git]' + subject_text << "[#{project.path_with_namespace}]" + subject_text << "[#{ref_name}]" if @action == :push + subject_text << ' ' + + if @action == :push && commits + if commits.length > 1 + subject_text << "Deleted " if reverse_compare? + subject_text << "#{commits.length} commits: #{commits.first.title}" + else + subject_text << "Deleted 1 commit: " if reverse_compare? + subject_text << commits.first.title + end + else + subject_action = action_name.dup + subject_action[0] = subject_action[0].capitalize + subject_text << "#{subject_action} #{ref_type} #{ref_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 496256700b8..403ebeec474 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -199,7 +199,7 @@ module Gitlab s = s.gsub(/^#/, "\\#") s = s.gsub(/^-/, "\\-") s = s.gsub("`", "\\~") - s = s.gsub("\r", "") + s = s.delete("\r") s = s.gsub("\n", " \n") s end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 8b1b6f48ed5..e0163499e30 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.safe_name, path: repo.path, namespace: namespace, diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index dd393fe09d2..07b856ca64c 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -16,6 +16,17 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return true unless exists? + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end + end + + private + + def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") # function will return true if succesful @@ -54,6 +65,12 @@ module Gitlab exit_status end + + def call_update_hook(gl_id, oldrev, newrev, ref) + Dir.chdir(repo_path) do + system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + end + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 270cbcd9ccd..74d1529e1ff 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -46,7 +46,7 @@ module Gitlab end def github_options - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitHub.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 9c00896c913..86fb6c51765 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -75,7 +75,7 @@ module Gitlab end def gitlab_options - OmniAuth::Strategies::GitLab.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index d9452de6a50..7baaadb813c 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["path"], description: repo["description"], diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb index cc9a91c91f4..8e22aa9286d 100644 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ b/lib/gitlab/gitorious_import/project_creator.rb @@ -10,7 +10,8 @@ module Gitlab end def execute - ::Projects::CreateService.new(current_user, + ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.path, description: repo.description, diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 87fee28dc01..62da327931f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -171,8 +171,6 @@ module Gitlab when /\AMilestone:/ "#fee3ff" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" when "Status: New" "#428bca" when "Status: Accepted" @@ -199,6 +197,8 @@ module Gitlab "#8e44ad" when "Type: Other" "#7f8c8d" + when *@closed_statuses.map { |s| nice_status_name(s) } + "#cfcfcf" else "#e2e2e2" end @@ -227,7 +227,7 @@ module Gitlab s = s.gsub("`", "\\`") # Carriage returns make me sad - s = s.gsub("\r", "") + s = s.delete("\r") # Markdown ignores single newlines, but we need them as <br />. s = s.gsub("\n", " \n") diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 1cb7d16aeb3..87821c23460 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.name, description: repo.summary, diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 16ff03c38d4..c438a3d167b 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -37,13 +37,15 @@ module Gitlab # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block unless user.blocked? + user.block false else user.activate if user.blocked? && !ldap_config.block_auto_created_users true end else + # Block the user if they no longer exist in LDAP/AD + user.block false end rescue diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 4202c786466..9d9617761b3 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -10,23 +10,9 @@ module Gitlab @request = request end - # Return a response for a download request - # Can be a response to: - # Request from a user to get the file - # Request from gitlab-workhorse which file to serve to the user - def render_download_hypermedia_response(oid) - render_response_to_download do - if check_download_accept_header? - render_lfs_download_hypermedia(oid) - else - render_not_found - end - end - end - def render_download_object_response(oid) render_response_to_download do - if check_download_sendfile_header? && check_download_accept_header? + if check_download_sendfile_header? render_lfs_sendfile(oid) else render_not_found @@ -34,20 +20,15 @@ module Gitlab end end - def render_lfs_api_auth - render_response_to_push do - request_body = JSON.parse(@request.body.read) - return render_not_found if request_body.empty? || request_body['objects'].empty? - - response = build_response(request_body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] + def render_batch_operation_response + request_body = JSON.parse(@request.body.read) + case request_body["operation"] + when "download" + render_batch_download(request_body) + when "upload" + render_batch_upload(request_body) + else + render_not_found end end @@ -71,13 +52,24 @@ module Gitlab end end + def render_unsupported_deprecated_api + [ + 501, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + private def render_not_enabled [ 501, { - "Content-Type" => "application/vnd.git-lfs+json", + "Content-Type" => "application/json; charset=utf-8", }, [JSON.dump({ 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', @@ -142,18 +134,35 @@ module Gitlab end end - def render_lfs_download_hypermedia(oid) - return render_not_found unless oid.present? + def render_batch_upload(body) + return render_not_found if body.empty? || body['objects'].nil? - lfs_object = object_for_download(oid) - if lfs_object + render_response_to_push do + response = build_upload_batch_response(body['objects']) [ 200, - { "Content-Type" => "application/vnd.git-lfs+json" }, - [JSON.dump(download_hypermedia(oid))] + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_batch_download(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_download do + response = build_download_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] ] - else - render_not_found end end @@ -199,10 +208,6 @@ module Gitlab @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" end - def check_download_accept_header? - @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8" - end - def user_can_fetch? # Check user access against the project they used to initiate the pull @user.can?(:download_code, @origin_project) @@ -215,7 +220,7 @@ module Gitlab def storage_project(project) if project.forked? - project.forked_from_project + storage_project(project.forked_from_project) else project end @@ -255,7 +260,7 @@ module Gitlab end def link_to_project(object) - if object && !object.projects.exists?(@project) + if object && !object.projects.exists?(@project.id) object.projects << @project object.save end @@ -266,42 +271,56 @@ module Gitlab @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set end - def build_response(objects) + def build_upload_batch_response(objects) selected_objects = select_existing_objects(objects) - upload_hypermedia(objects, selected_objects) + upload_hypermedia_links(objects, selected_objects) end - def download_hypermedia(oid) - { - '_links' => { - 'download' => - { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", - 'header' => { - 'Accept' => "application/vnd.git-lfs+json; charset=utf-8", - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - } + def build_download_batch_response(objects) + selected_objects = select_existing_objects(objects) + + download_hypermedia_links(objects, selected_objects) end - def upload_hypermedia(all_objects, existing_objects) + def download_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| - object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) + if existing_objects.include?(object['oid']) + object['actions'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + else + object['error'] = { + 'code' => 404, + 'message' => "Object does not exist on the server or you don't have permissions to access it", + } + end end { 'objects' => all_objects } end - def hypermedia_links(object) - { - "upload" => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", - 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } - }.compact - } + def upload_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + # generate actions only for non-existing objects + next if existing_objects.include?(object['oid']) + + object['actions'] = { + 'upload' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + end + + { 'objects' => all_objects } end end end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 4809e834984..78d02891102 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -34,7 +34,7 @@ module Gitlab case path_match[1] when "info/lfs" - lfs.render_download_hypermedia_response(oid) + lfs.render_unsupported_deprecated_api when "gitlab-lfs" lfs.render_download_object_response(oid) else @@ -48,7 +48,9 @@ module Gitlab # Check for Batch API if post_path[0].ends_with?("/info/lfs/objects/batch") - lfs.render_lfs_api_auth + lfs.render_batch_operation_response + elsif post_path[0].ends_with?("/info/lfs/objects") + lfs.render_unsupported_deprecated_api else nil end diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb deleted file mode 100644 index b082bfc434b..00000000000 --- a/lib/gitlab/markdown.rb +++ /dev/null @@ -1,200 +0,0 @@ -require 'html/pipeline' - -module Gitlab - # Custom parser for GitLab-flavored Markdown - # - # See the files in `lib/gitlab/markdown/` for specific processing information. - module Markdown - # Convert a Markdown String into an HTML-safe String of HTML - # - # Note that while the returned HTML will have been sanitized of dangerous - # HTML, it may post a risk of information leakage if it's not also passed - # through `post_process`. - # - # Also note that the returned String is always HTML, not XHTML. Views - # requiring XHTML, such as Atom feeds, need to call `post_process` on the - # result, providing the appropriate `pipeline` option. - # - # markdown - Markdown String - # context - Hash of context options passed to our HTML Pipeline - # - # Returns an HTML-safe String - def self.render(markdown, context = {}) - html = renderer.render(markdown) - html = gfm(html, context) - - html.html_safe - end - - # Convert a Markdown String into HTML without going through the HTML - # Pipeline. - # - # Note that because the pipeline is skipped, SanitizationFilter is as well. - # Do not output the result of this method to the user. - # - # markdown - Markdown String - # - # Returns a String - def self.render_without_gfm(markdown) - renderer.render(markdown) - end - - # Perform post-processing on an HTML String - # - # This method is used to perform state-dependent changes to a String of - # HTML, such as removing references that the current user doesn't have - # permission to make (`RedactorFilter`). - # - # html - String to process - # options - Hash of options to customize output - # :pipeline - Symbol pipeline type - # :project - Project - # :user - User object - # - # Returns an HTML-safe String - def self.post_process(html, options) - context = { - project: options[:project], - current_user: options[:user] - } - doc = post_processor.to_document(html, context) - - if options[:pipeline] == :atom - doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) - else - doc.to_html - end.html_safe - end - - # Provide autoload paths for filters to prevent a circular dependency error - autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' - autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' - autoload :CommitReferenceFilter, 'gitlab/markdown/commit_reference_filter' - autoload :EmojiFilter, 'gitlab/markdown/emoji_filter' - autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter' - autoload :ExternalLinkFilter, 'gitlab/markdown/external_link_filter' - autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter' - autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter' - autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter' - autoload :RedactorFilter, 'gitlab/markdown/redactor_filter' - autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' - autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' - autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' - autoload :SyntaxHighlightFilter, 'gitlab/markdown/syntax_highlight_filter' - autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' - autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' - autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' - autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter' - - # Public: Parse the provided HTML with GitLab-Flavored Markdown - # - # html - HTML String - # options - A Hash of options used to customize output (default: {}) - # :no_header_anchors - Disable header anchors in TableOfContentsFilter - # :path - Current path String - # :pipeline - Symbol pipeline type - # :project - Current Project object - # :project_wiki - Current ProjectWiki object - # :ref - Current ref String - # - # Returns an HTML-safe String - def self.gfm(html, options = {}) - return '' unless html.present? - - @pipeline ||= HTML::Pipeline.new(filters) - - context = { - # SanitizationFilter - pipeline: options[:pipeline], - - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url, - - # ReferenceFilter - only_path: only_path_pipeline?(options[:pipeline]), - project: options[:project], - - # RelativeLinkFilter - project_wiki: options[:project_wiki], - ref: options[:ref], - requested_path: options[:path], - - # TableOfContentsFilter - no_header_anchors: options[:no_header_anchors] - } - - @pipeline.to_html(html, context).html_safe - end - - private - - # Check if a pipeline enables the `only_path` context option - # - # Returns Boolean - def self.only_path_pipeline?(pipeline) - case pipeline - when :atom, :email - false - else - true - end - end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - def self.renderer - @markdown ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) - end - end - - def self.post_processor - @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter]) - end - - # Filters used in our pipeline - # - # SanitizationFilter should come first so that all generated reference HTML - # goes through untouched. - # - # See https://github.com/jch/html-pipeline#filters for more filters. - def self.filters - [ - Gitlab::Markdown::SyntaxHighlightFilter, - Gitlab::Markdown::SanitizationFilter, - - Gitlab::Markdown::UploadLinkFilter, - Gitlab::Markdown::RelativeLinkFilter, - Gitlab::Markdown::EmojiFilter, - Gitlab::Markdown::TableOfContentsFilter, - Gitlab::Markdown::AutolinkFilter, - Gitlab::Markdown::ExternalLinkFilter, - - Gitlab::Markdown::UserReferenceFilter, - Gitlab::Markdown::IssueReferenceFilter, - Gitlab::Markdown::ExternalIssueReferenceFilter, - Gitlab::Markdown::MergeRequestReferenceFilter, - Gitlab::Markdown::SnippetReferenceFilter, - Gitlab::Markdown::CommitRangeReferenceFilter, - Gitlab::Markdown::CommitReferenceFilter, - Gitlab::Markdown::LabelReferenceFilter, - - Gitlab::Markdown::TaskListFilter - ] - end - end -end diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb deleted file mode 100644 index fd5b7eb9332..00000000000 --- a/lib/gitlab/markdown/abstract_reference_filter.rb +++ /dev/null @@ -1,100 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # Issues, Snippets and Merge Requests shares similar functionality in refernce filtering. - # All this functionality moved to this class - class AbstractReferenceFilter < ReferenceFilter - include CrossProjectReference - - def self.object_class - # Implement in child class - # Example: MergeRequest - end - - def self.object_name - object_class.name.underscore - end - - def self.object_sym - object_name.to_sym - end - - def self.data_reference - "data-#{object_name.dasherize}" - end - - # Public: Find references in text (like `!123` for merge requests) - # - # AnyReferenceFilter.references_in(text) do |match, object| - # "<a href=...>PREFIX#{object}</a>" - # end - # - # PREFIX - symbol that detects reference (like ! for merge requests) - # object - reference object (snippet, merget request etc) - # text - String text to search. - # - # Yields the String match, the Integer referenced object ID, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(object_class.reference_pattern) do |match| - yield match, $~[object_sym].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } - end - - delegate :object_class, :object_sym, :references_in, to: :class - - def find_object(project, id) - # Implement in child class - # Example: project.merge_requests.find - end - - def url_for_object(object, project) - # Implement in child class - # Example: project_merge_request_url - end - - def call - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content) - end - end - - # Replace references (like `!123` for merge requests) in text with links - # to the referenced object's details page. - # - # text - String text to replace references in. - # - # Returns a String with references replaced with links. All links - # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text) - references_in(text) do |match, id, project_ref| - project = project_from_ref(project_ref) - - if project && object = find_object(project, id) - title = escape_once("#{object_title}: #{object.title}") - klass = reference_class(object_sym) - data = data_attribute(project: project.id, object_sym => object.id) - url = url_for_object(object, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end - end - - def object_title - object_class.name.titleize - end - end - end -end diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb deleted file mode 100644 index e070edae0a4..00000000000 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ /dev/null @@ -1,92 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces commit range references with links. - # - # This filter supports cross-project references. - class CommitRangeReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find commit range references in text - # - # CommitRangeReferenceFilter.references_in(text) do |match, commit_range, project_ref| - # "<a href=...>#{commit_range}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit range, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(CommitRange.reference_pattern) do |match| - yield match, $~[:commit_range], $~[:project] - end - end - - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit-range") - range = CommitRange.new(id, project) - - return unless range.valid_commits? - - { commit_range: range } - end - - def initialize(*args) - super - - @commit_map = {} - end - - def call - replace_text_nodes_matching(CommitRange.reference_pattern) do |content| - commit_range_link_filter(content) - end - end - - # Replace commit range references in text with links to compare the commit - # ranges. - # - # text - String text to replace references in. - # - # Returns a String with commit range references replaced with links. All - # links have `gfm` and `gfm-commit_range` class names attached for - # styling. - def commit_range_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - range = CommitRange.new(id, project) - - if range.valid_commits? - url = url_for_commit_range(project, range) - - title = range.reference_title - klass = reference_class(:commit_range) - data = data_attribute(project: project.id, commit_range: id) - - project_ref += '@' if project_ref - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{range}</a>) - else - match - end - end - end - - def url_for_commit_range(project, range) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_compare_url(project.namespace, project, - range.to_param.merge(only_path: context[:only_path])) - end - end - end -end diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb deleted file mode 100644 index 8cdbeb1f9cf..00000000000 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ /dev/null @@ -1,88 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces commit references with links. - # - # This filter supports cross-project references. - class CommitReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find commit references in text - # - # CommitReferenceFilter.references_in(text) do |match, commit, project_ref| - # "<a href=...>#{commit}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit identifier, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Commit.reference_pattern) do |match| - yield match, $~[:commit], $~[:project] - end - end - - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit") - commit = commit_from_ref(project, id) - - return unless commit - - { commit: commit } - end - - def call - replace_text_nodes_matching(Commit.reference_pattern) do |content| - commit_link_filter(content) - end - end - - # Replace commit references in text with links to the commit specified. - # - # text - String text to replace references in. - # - # Returns a String with commit references replaced with links. All links - # have `gfm` and `gfm-commit` class names attached for styling. - def commit_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if commit = self.class.commit_from_ref(project, id) - url = url_for_commit(project, commit) - - title = escape_once(commit.link_title) - klass = reference_class(:commit) - data = data_attribute(project: project.id, commit: id) - - project_ref += '@' if project_ref - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{commit.short_id}</a>) - else - match - end - end - end - - def self.commit_from_ref(project, id) - if project && project.valid_repo? - project.commit(id) - end - end - - def url_for_commit(project, commit) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_commit_url(project.namespace, project, commit, - only_path: context[:only_path]) - end - end - end -end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb deleted file mode 100644 index 6ab04a584b0..00000000000 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # Common methods for ReferenceFilters that support an optional cross-project - # reference. - module CrossProjectReference - # Given a cross-project reference string, get the Project record - # - # Defaults to value of `context[:project]` if: - # * No reference is given OR - # * Reference given doesn't exist - # - # ref - String reference. - # - # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) - return context[:project] unless ref - - Project.find_with_namespace(ref) - end - end - end -end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb new file mode 100644 index 00000000000..8f3f43c0e91 --- /dev/null +++ b/lib/gitlab/markdown/pipeline.rb @@ -0,0 +1,34 @@ +require 'banzai' + +module Gitlab + module Markdown + class Pipeline + def self.[](name) + name ||= :full + const_get("#{name.to_s.camelize}Pipeline") + end + + def self.filters + [] + end + + def self.transform_context(context) + context + end + + def self.html_pipeline + @html_pipeline ||= HTML::Pipeline.new(filters) + end + + class << self + %i(call to_document to_html).each do |meth| + define_method(meth) do |text, context| + context = transform_context(context) + + html_pipeline.send(meth, text, context) + end + end + end + end + end +end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index d94b104bbf8..ba31599432b 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -62,7 +62,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].parameterize + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index fa068d50763..4f9cdef3869 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -18,10 +18,7 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum, - # added: ["CHANGELOG"], - # modified: [], - # removed: ["tmp/file.txt"] + # total_commits_count: Fixnum # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) @@ -33,11 +30,12 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. - commit_attrs = commits_limited.map(&:hook_attrs) + commit_attrs = commits_limited.map do |commit| + commit.hook_attrs(with_changed_files: true) + end type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" - repo_changes = repo_changes(project, newrev, oldrev) # Hash to be passed as post_receive_data data = { object_kind: type, @@ -60,10 +58,7 @@ module Gitlab visibility_level: project.visibility_level }, commits: commit_attrs, - total_commits_count: commits_count, - added: repo_changes[:added], - modified: repo_changes[:modified], - removed: repo_changes[:removed] + total_commits_count: commits_count } data @@ -94,27 +89,6 @@ module Gitlab newrev end end - - def repo_changes(project, newrev, oldrev) - changes = { added: [], modified: [], removed: [] } - compare_result = CompareService.new. - execute(project, newrev, project, oldrev) - - if compare_result - compare_result.diffs.each do |diff| - case true - when diff.deleted_file - changes[:removed] << diff.old_path - when diff.renamed_file, diff.new_file - changes[:added] << diff.new_path - else - changes[:modified] << diff.new_path - end - end - end - - changes - end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index da8df8a3025..42f7c26f3c4 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,73 +1,27 @@ -require 'gitlab/markdown' +require 'banzai' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. - class ReferenceExtractor - attr_accessor :project, :current_user, :load_lazy_references + class ReferenceExtractor < Banzai::ReferenceExtractor + attr_accessor :project, :current_user - def initialize(project, current_user = nil, load_lazy_references: true) + def initialize(project, current_user = nil) @project = project @current_user = current_user - @load_lazy_references = load_lazy_references - end - def analyze(text) - references.clear - @text = Gitlab::Markdown.render_without_gfm(text) - end + @references = {} - %i(user label issue merge_request snippet commit commit_range).each do |type| - define_method("#{type}s") do - references[type] - end + super() end - private - - def references - @references ||= Hash.new do |references, type| - type = type.to_sym - next references[type] if references.has_key?(type) - - references[type] = pipeline_result(type) - end + def analyze(text, context = {}) + super(text, context.merge(project: project)) end - # Instantiate and call HTML::Pipeline with a single reference filter type, - # returning the result - # - # filter_type - Symbol reference type (e.g., :commit, :issue, etc.) - # - # Returns the results Array for the requested filter type - def pipeline_result(filter_type) - return [] if @text.blank? - - klass = "#{filter_type.to_s.camelize}ReferenceFilter" - filter = Gitlab::Markdown.const_get(klass) - - context = { - project: project, - current_user: current_user, - - # We don't actually care about the links generated - only_path: true, - ignore_blockquotes: true, - - # ReferenceGathererFilter - load_lazy_references: false, - reference_filter: filter - } - - pipeline = HTML::Pipeline.new([filter, Gitlab::Markdown::ReferenceGathererFilter], context) - result = pipeline.call(@text) - - values = result[:references][filter_type].uniq - - if @load_lazy_references - values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq + %i(user label issue merge_request snippet commit commit_range).each do |type| + define_method("#{type}s") do + @references[type] ||= references(type, project: project, current_user: current_user) end - - values end end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 31aa3528c4c..2ef0e982256 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -14,7 +14,7 @@ module Gitlab def self.mute_mailer code = <<-eos -def Notify.delay +def Notify.deliver_later self end eos diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index d87a4c9bb4a..3489fb251b6 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -36,6 +36,11 @@ module Gitlab @duration ||= started_at && finished_at ? finished_at - started_at : 0 end + # Returns the total query duration in seconds. + def query_duration + @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 + end + def to_param @id end diff --git a/lib/omni_auth/request_forgery_protection.rb b/lib/omni_auth/request_forgery_protection.rb index 3557522d3c9..69155131d8d 100644 --- a/lib/omni_auth/request_forgery_protection.rb +++ b/lib/omni_auth/request_forgery_protection.rb @@ -1,66 +1,21 @@ # Protects OmniAuth request phase against CSRF. module OmniAuth - # Based on ActionController::RequestForgeryProtection. - class RequestForgeryProtection - def initialize(env) - @env = env - end - - def request - @request ||= ActionDispatch::Request.new(@env) - end - - def session - request.session - end - - def reset_session - request.reset_session - end - - def params - request.params - end - - def call - verify_authenticity_token - end + module RequestForgeryProtection + class Controller < ActionController::Base + protect_from_forgery with: :exception - def verify_authenticity_token - if !verified_request? - Rails.logger.warn "Can't verify CSRF token authenticity" if Rails.logger - handle_unverified_request + def index + head :ok end end - private - - def protect_against_forgery? - ApplicationController.allow_forgery_protection - end - - def request_forgery_protection_token - ApplicationController.request_forgery_protection_token - end - - def forgery_protection_strategy - ApplicationController.forgery_protection_strategy - end - - def verified_request? - !protect_against_forgery? || request.get? || request.head? || - form_authenticity_token == params[request_forgery_protection_token] || - form_authenticity_token == request.headers['X-CSRF-Token'] - end - - def handle_unverified_request - forgery_protection_strategy.new(self).handle_unverified_request + def self.app + @app ||= Controller.action(:index) end - # Sets the token value for the current session. - def form_authenticity_token - session[:_csrf_token] ||= SecureRandom.base64(32) + def self.call(env) + app.call(env) end end end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 6762ca47c32..8c309efc7b8 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -39,7 +39,7 @@ module Rouge lineanchorsid: 'L', anchorlinenos: false, inline_theme: nil - ) + ) @nowrap = nowrap @cssclass = cssclass @linenos = linenos diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index f0a6c2b30e9..aa2da92c138 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -33,12 +33,13 @@ app_user="git" app_root="/home/$app_user/gitlab" pid_path="$app_root/tmp/pids" socket_path="$app_root/tmp/sockets" +rails_socket="$socket_path/gitlab.socket" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" shell_path="/bin/bash" @@ -327,7 +328,7 @@ print_status() { printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 79ae8e0ae55..7fc495db545 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -36,7 +36,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # '-listenNetwork tcp -listenAddr localhost:8181'. # The -authBackend setting tells gitlab-workhorse where it can reach # Unicorn. -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 0cf5292b290..fc5475c4eef 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -10,34 +10,12 @@ ## If you change this file in a Merge Request, please also create ## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## -################################## -## CHUNKED TRANSFER ## -################################## -## -## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] -## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object -## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get -## around this by tweaking this configuration file and either: -## - installing an old version of Nginx with the chunkin module [2] compiled in, or -## - using a newer version of Nginx. -## -## At the time of writing we do not know if either of these theoretical solutions works. -## As a workaround users can use Git over SSH to push large files. -## -## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 -## [1] https://github.com/agentzh/chunkin-nginx-module#status -## [2] https://github.com/agentzh/chunkin-nginx-module -## ################################### ## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. -upstream gitlab { - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; -} - upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -54,10 +32,6 @@ server { server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; - ## Increase this if you want to upload large attachments - ## Or if you want to accept large git objects over http - client_max_body_size 20m; - ## See app/controllers/application_controller.rb for headers set ## Individual nginx logs for this GitLab vhost @@ -65,97 +39,8 @@ server { error_log /var/log/nginx/gitlab_error.log; location / { - ## Serve static files from defined root folder. - ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri /index.html $uri.html @gitlab; - } - - ## We route uploads through GitLab to prevent XSS and enforce access control. - location /uploads/ { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - ## If a file, which is not found in the root folder is requested, - ## then the proxy passes the request to the upsteam (gitlab unicorn). - location @gitlab { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/api/v3/projects/.*/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ /ci/api/v1/builds/[0-9]+/artifacts { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location @gitlab-workhorse { client_max_body_size 0; - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; + gzip off; ## https://github.com/gitlabhq/gitlabhq/issues/694 ## Some requests take more than 30 seconds. @@ -163,14 +48,7 @@ server { proxy_connect_timeout 300; proxy_redirect off; - # Do not buffer Git HTTP responses - proxy_buffering off; - - # The following settings only work with NGINX 1.7.11 or newer - # - # # Pass chunked request bodies to gitlab-workhorse as-is - # proxy_request_buffering off; - # proxy_http_version 1.1; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -179,18 +57,4 @@ server { proxy_pass http://gitlab-workhorse; } - - ## Enable gzip compression as per rails guide: - ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression - ## WARNING: If you are using relative urls remove the block below - ## See config/application.rb under "Relative url support" for the list of - ## other files that need to be changed for relative url support - location ~ ^/(assets)/ { - root /home/git/gitlab/public; - gzip_static on; # to serve pre-gzipped version - expires max; - add_header Cache-Control public; - } - - error_page 502 /502.html; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 31a651c87fd..1e5f85413ec 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -14,34 +14,12 @@ ## If you change this file in a Merge Request, please also create ## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## -################################## -## CHUNKED TRANSFER ## -################################## -## -## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] -## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object -## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get -## around this by tweaking this configuration file and either: -## - installing an old version of Nginx with the chunkin module [2] compiled in, or -## - using a newer version of Nginx. -## -## At the time of writing we do not know if either of these theoretical solutions works. -## As a workaround users can use Git over SSH to push large files. -## -## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 -## [1] https://github.com/agentzh/chunkin-nginx-module#status -## [2] https://github.com/agentzh/chunkin-nginx-module -## ################################### ## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. -upstream gitlab { - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; -} - upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -56,12 +34,11 @@ server { listen [::]:80 ipv6only=on default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - return 301 https://$server_name$request_uri; + return 301 https://$http_host$request_uri; access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; } - ## HTTPS host server { listen 0.0.0.0:443 ssl; @@ -70,10 +47,6 @@ server { server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; - ## Increase this if you want to upload large attachments - ## Or if you want to accept large git objects over http - client_max_body_size 20m; - ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ ssl on; @@ -110,98 +83,7 @@ server { error_log /var/log/nginx/gitlab_error.log; location / { - ## Serve static files from defined root folder. - ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri /index.html $uri.html @gitlab; - } - - ## We route uploads through GitLab to prevent XSS and enforce access control. - location /uploads/ { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Ssl on; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - ## If a file, which is not found in the root folder is requested, - ## then the proxy passes the request to the upsteam (gitlab unicorn). - location @gitlab { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Ssl on; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/api/v3/projects/.*/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ /ci/api/v1/builds/[0-9]+/artifacts { - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location @gitlab-workhorse { client_max_body_size 0; - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. gzip off; ## https://github.com/gitlabhq/gitlabhq/issues/694 @@ -210,14 +92,7 @@ server { proxy_connect_timeout 300; proxy_redirect off; - # Do not buffer Git HTTP responses - proxy_buffering off; - - # The following settings only work with NGINX 1.7.11 or newer - # - # # Pass chunked request bodies to gitlab-workhorse as-is - # proxy_request_buffering off; - # proxy_http_version 1.1; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -226,18 +101,4 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://gitlab-workhorse; } - - ## Enable gzip compression as per rails guide: - ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression - ## WARNING: If you are using relative urls remove the block below - ## See config/application.rb under "Relative url support" for the list of - ## other files that need to be changed for relative url support - location ~ ^/(assets)/ { - root /home/git/gitlab/public; - gzip_static on; # to serve pre-gzipped version - expires max; - add_header Cache-Control public; - } - - error_page 502 /502.html; } diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake deleted file mode 100644 index 49435504c67..00000000000 --- a/lib/tasks/ci/schedule_builds.rake +++ /dev/null @@ -1,6 +0,0 @@ -namespace :ci do - desc "GitLab CI | Clean running builds" - task schedule_builds: :environment do - Ci::Scheduler.new.perform - end -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3c46bcea40e..cb4abe13799 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:lfs:create"].invoke backup = Backup::Manager.new backup.pack @@ -34,6 +35,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") + Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -134,6 +136,25 @@ namespace :gitlab do end end + namespace :lfs do + task create: :environment do + $progress.puts "Dumping lfs objects ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("lfs") + $progress.puts "[SKIPPED]".cyan + else + Backup::Lfs.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring lfs objects ... ".blue + Backup::Lfs.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a25fac62cfc..0469c5a61c3 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -331,7 +331,7 @@ namespace :gitlab do end def check_redis_version - min_redis_version = "2.4.0" + min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) @@ -822,10 +822,27 @@ namespace :gitlab do namespace_dirs.each do |namespace_dir| repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each do |dir| - puts "\nChecking repo at #{dir}" - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir) - end + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end + end + end + + namespace :user do + desc "GitLab | Check the integrity of a specific user's repositories" + task :check_repos, [:username] => :environment do |t, args| + username = args[:username] || prompt("Check repository integrity for which username? ".blue) + user = User.find_by(username: username) + if user + repo_dirs = user.authorized_projects.map do |p| + File.join( + Gitlab.config.gitlab_shell.repos_path, + "#{p.path_with_namespace}.git" + ) + end + + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + else + puts "\nUser '#{username}' not found".red end end end @@ -952,4 +969,35 @@ namespace :gitlab do false end end + + def check_repo_integrity(repo_dir) + puts "\nChecking repo at #{repo_dir.yellow}" + + git_fsck(repo_dir) + check_config_lock(repo_dir) + check_ref_locks(repo_dir) + end + + def git_fsck(repo_dir) + puts "Running `git fsck`".yellow + system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) + end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir,'config.lock')) + config_output = config_exists ? 'yes'.red : 'no'.green + puts "'config.lock' file exists?".yellow + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) + if lock_files.present? + puts "Ref lock files exist:".red + lock_files.each do |lock_file| + puts " #{lock_file}" + end + else + puts "No ref lock files exist".green + end + end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake new file mode 100644 index 00000000000..65ee430d550 --- /dev/null +++ b/lib/tasks/gitlab/git.rake @@ -0,0 +1,55 @@ +namespace :gitlab do + namespace :git do + + desc "GitLab | Git | Repack" + task repack: :environment do + failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + desc "GitLab | Git | Run garbage collection on all repos" + task gc: :environment do + failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + desc "GitLab | Git | Prune all repos" + task prune: :environment do + failures = perform_git_cmd(%W(git prune), "Git Prune") + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + def perform_git_cmd(cmd, message) + puts "Starting #{message} on all repositories" + + failures = [] + all_repos do |repo| + if system(*cmd, chdir: repo) + puts "Performed #{message} at #{repo}" + else + failures << repo + end + end + + failures + end + + def output_failures(failures) + puts "The following repositories reported errors:".red + failures.each { |f| puts "- #{f}" } + end + + end +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index c1ee271ae2b..1c04f47f08f 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -64,6 +64,8 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".green + project.update_repository_size + project.update_commit_count else puts " * Failed trying to create #{project.name} (#{repo_path})".red puts " Errors: #{project.errors.messages}".red diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake new file mode 100644 index 00000000000..c7596e7abcb --- /dev/null +++ b/lib/tasks/gitlab/list_repos.rake @@ -0,0 +1,17 @@ +namespace :gitlab do + task list_repos: :environment do + scope = Project + if ENV['SINCE'] + date = Time.parse(ENV['SINCE']) + warn "Listing repositories with activity or changes since #{date}" + project_ids = Project.where('last_activity_at > ? OR updated_at > ?', date, date).pluck(:id).sort + namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort + scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) + end + scope.find_each do |project| + base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + puts base + '.git' + puts base + '.wiki.git' + end + end +end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index c95b6540ebc..ebe516ec879 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,16 +2,6 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end -end - namespace :gitlab do # Ask if the user wants to continue @@ -103,7 +93,7 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}" + puts " Warning ".colorize(:black).on_yellow puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." puts " For correct results you should run this as user #{gitlab_user.magenta}." @@ -128,4 +118,12 @@ namespace :gitlab do false end end + + def all_repos + IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 365ff2defd4..0985ef3a669 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -19,6 +19,33 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run model specs' + task :models do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @models) + ] + run_commands(cmds) + end + + desc 'GitLab | Rspec | Run service specs' + task :services do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @services) + ] + run_commands(cmds) + end + + desc 'GitLab | Rspec | Run lib specs' + task :lib do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @lib) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run benchmark specs' task :benchmark do cmds = [ @@ -32,7 +59,7 @@ namespace :spec do task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services --tag ~@benchmark) ] run_commands(cmds) end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index d5a96fd38f4..3acfc6e2075 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -1,11 +1,31 @@ Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach') namespace :spinach do + namespace :project do + desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" + task :half do + cmds = [ + %W(rake gitlab:setup), + %W(spinach --tags @project_commits,@project_issues,@project_merge_requests), + ] + run_commands(cmds) + end + + desc "GitLab | Spinach | Run remaining project spinach features" + task :rest do + cmds = [ + %W(rake gitlab:setup), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests), + ] + run_commands(cmds) + end + end + desc "GitLab | Spinach | Run project spinach features" task :project do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), ] run_commands(cmds) end @@ -14,7 +34,7 @@ namespace :spinach do task :other do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits), + %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), ] run_commands(cmds) end @@ -33,4 +53,4 @@ def run_commands(cmds) cmds.each do |cmd| system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end -end
\ No newline at end of file +end |