diff options
author | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-01-14 10:58:40 +0100 |
---|---|---|
committer | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-01-14 10:58:40 +0100 |
commit | 57a67c4fdb147b4b9eee9f8de35004b86fe9ed12 (patch) | |
tree | 43c2118bea0b973bb5ac6acb7a2b910ed9cab527 /lib | |
parent | 633ac13404a38bfa3901f515179dc795cded2fa9 (diff) | |
parent | d139276a3757d3df45232b557e1bb388d21268d0 (diff) | |
download | gitlab-ce-57a67c4fdb147b4b9eee9f8de35004b86fe9ed12.tar.gz |
Merge branch '8-4-stable' into ci/api-triggers
* 8-4-stable: (280 commits)
Add Changelog entry for build traces data integrity fix
Update doc_styleguide.md [ci skip]
Added hint that you can search users by name, username, or email.
Add changelog
Version 8.4.0.rc1
Randomize metrics sample intervals
Make the metrics sampler interval configurable
Don't automatically require awesome_print
Disable colorization if STDOUT is not a tty
Block the reported user before destroying the record
changes `$quote-gray` to `$secondary-text`
makes message plural for multiple MRs and removes from loop. Duh.
Prepare Installation and Update docs for 8.4 RC1
Mention channel/key bug in irkerd docs
Revert "Remove the `:coffee` and `:coffeescript` Haml filters"
gets merge request discussion working again
adds back in discussion.haml.html for issues commenting and closing/reopening properly.
removing last chunk of MR ajax changes, rest will be in another MR
reverting more MR ajax files, will appear in different commit
reverting MR ajax changes, which will be in a different MR
...
Diffstat (limited to 'lib')
45 files changed, 598 insertions, 246 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1108277aabf..00c933237fe 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3c1c6bda260..5c97fe1c88c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -211,7 +211,7 @@ module API unauthorized! unless merge_request.can_be_merged_by?(current_user) not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged if merge_request.unchecked? + merge_request.check_if_can_be_merged render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a9e0960872a..71bb342f844 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,7 +3,7 @@ module API class Projects < Grape::API before { authenticate! } - resource :projects do + resource :projects, requirements: { id: /[^\/]+/ } do helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -118,7 +119,8 @@ module API attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -163,7 +165,8 @@ module API attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -182,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -229,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end @@ -269,7 +274,7 @@ module API # Remove a forked_from relationship # # Parameters: - # id: (required) - The ID of the project being marked as a fork + # id: (required) - The ID of the project being marked as a fork # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do @@ -278,6 +283,16 @@ module API user_project.forked_project_link.destroy end end + + # Upload a file + # + # Parameters: + # id: (required) - The ID of the project + # file: (required) - The file to be uploaded + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + # search for projects current_user has access to # # Parameters: diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 47621f443e6..2d8a9e51bb9 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -40,6 +40,27 @@ module API end end + # Delete tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # DELETE /projects/:id/repository/tags/:tag + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + authorize_push_project + result = DeleteTagService.new(user_project, current_user). + execute(params[:tag_name]) + + if result[:status] == :success + { + tag_name: params[:tag_name] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + # Add release notes to tag # # Parameters: diff --git a/lib/api/users.rb b/lib/api/users.rb index 3400f0713ef..0d7813428e2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,7 +39,7 @@ module API if current_user.is_admin? present @user, with: Entities::UserFull else - present @user, with: Entities::UserBasic + present @user, with: Entities::User end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 63ad8910c0f..b2db10e6864 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -47,7 +47,17 @@ module Banzai { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } end - delegate :object_class, :object_sym, :references_in, to: :class + def object_class + self.class.object_class + end + + def object_sym + self.class.object_sym + end + + def references_in(*args, &block) + self.class.references_in(*args, &block) + end def find_object(project, id) # Implement in child class @@ -60,27 +70,31 @@ module Banzai end def call - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + if object_class.reference_pattern + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end - # `[Issue](#123)`, which is turned into - # `<a href="#123">Issue</a>` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) + # `[Issue](#123)`, which is turned into + # `<a href="#123">Issue</a>` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end end - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + if object_class.link_reference_pattern + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb new file mode 100644 index 00000000000..e88b27c1fae --- /dev/null +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -0,0 +1,22 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces milestone references with links. + class MilestoneReferenceFilter < AbstractReferenceFilter + def self.object_class + Milestone + end + + def find_object(project, id) + project.milestones.find_by(iid: id) + end + + def url_for_object(issue, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_milestone_url(project.namespace, project, milestone, + only_path: context[:only_path]) + end + end + end +end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index f01a32b5ae5..66f77902319 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -10,7 +10,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 8ca05ace88c..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -124,7 +124,7 @@ module Banzai def replace_link_nodes_with_text(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -162,7 +162,7 @@ module Banzai def replace_link_nodes_with_href(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 12412ff7ea9..bef04112919 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -16,7 +16,7 @@ module Banzai end def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| gather_references(node) end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5a081125f21..66f166939e4 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -91,7 +91,7 @@ module Banzai parts = request_path.split('/') parts.pop if path_type(request_path) != 'tree' - while parts.length > 1 && path.start_with?('../') + while path.start_with?('../') parts.pop path.sub!('../', '') end diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index bdf7c2ebdfc..d0ce13003a5 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -12,13 +12,18 @@ module Banzai # # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class(node, *new_class_names) + def add_css_class_with_fix(node, *new_class_names) if new_class_names.include?('task-list') - super if node.children.any? { |c| c['class'] == 'task-list-item' } - else - super + # Don't add class to all lists + return + elsif new_class_names.include?('task-list-item') + add_css_class_without_fix(node.parent, 'task-list') end + + add_css_class_without_fix(node, *new_class_names) end + + alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 38750b55ec7..838155e8831 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -22,6 +22,7 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, + Filter::MilestoneReferenceFilter, Filter::TaskListFilter ] diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 6725c9039a9..a3c9d4f43aa 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,7 +3,23 @@ require 'banzai' module Banzai module Pipeline class SingleLinePipeline < GfmPipeline + def self.filters + @filters ||= [ + Filter::SanitizationFilter, + Filter::EmojiFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter, + + Filter::UserReferenceFilter, + Filter::IssueReferenceFilter, + Filter::ExternalIssueReferenceFilter, + Filter::MergeRequestReferenceFilter, + Filter::SnippetReferenceFilter, + Filter::CommitRangeReferenceFilter, + Filter::CommitReferenceFilter, + ] + end end end end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb new file mode 100644 index 00000000000..1e1b51e683e --- /dev/null +++ b/lib/banzai/querying.rb @@ -0,0 +1,18 @@ +module Banzai + module Querying + # Searches a Nokogiri document using a CSS query, optionally optimizing it + # whenever possible. + # + # document - A document/element to search. + # query - The CSS query to use. + # + # Returns a Nokogiri::XML::NodeSet. + def self.css(document, query) + # When using "a.foo" Nokogiri compiles this to "//a[...]" but + # "descendant::a[...]" is quite a bit faster and achieves the same result. + xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + + document.xpath(xpath) + end + end +end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..891c0fd7749 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,7 +18,7 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED + if cache_key Rails.cache.fetch(cache_key) do cacheless_render(text, context) end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 459e3d6bcdb..4c15d58d680 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -150,6 +150,18 @@ module Gitlab "#{path}.git", tag_name]) end + # Gc repository + # + # path - project path with namespace + # + # Ex. + # gc("gitlab/gitlab-ci") + # + def gc(path) + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', + "#{path}.git"]) + end + # Add new key to gitlab-shell # # Ex. diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 86bfa0a4378..34e949130da 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -23,6 +23,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_allow_failure: build.allow_failure, # TODO: do we still need it? project_id: project.id, diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7a86c09158e..7f938780ab1 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -41,6 +41,9 @@ module Gitlab use_db && ActiveRecord::Base.connection.active? && !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') + + rescue ActiveRecord::NoDatabaseError + false end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2b252b32887..2ca21af5bc8 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -74,7 +74,7 @@ module Gitlab def sent_notification return nil unless reply_key - + SentNotification.for(reply_key) end @@ -82,10 +82,7 @@ module Gitlab attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) attachments.each do |link| - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - reply << "\n\n#{text}" + reply << "\n\n#{link[:markdown]}" end reply diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 403ebeec474..db580b5e578 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,9 +232,7 @@ module Gitlab return nil if res.nil? - text = "[#{res['alt']}](#{res['url']})" - text = "!#{text}" if res['is_image'] - text + res[:markdown] end def build_attachment_url(rel_url) diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b5720f6e2cb..2b0afbc7b39 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -12,39 +12,59 @@ module Gitlab end def execute - #Issues && Comments + import_issues + import_pull_requests + + true + end + + private + + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? - - body = @formatter.author_line(issue.user.login) - body += issue.body || "" + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - if issue.comments > 0 - body += @formatter.comments_header + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end + end + end + end + + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| + pull_request = PullRequestFormatter.new(project, raw_data) - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) - ) + if !pull_request.cross_project? && pull_request.valid? + merge_request = MergeRequest.create!(pull_request.attributes) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) end end end - private + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) + end + + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..b7c47958cc7 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,101 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def cross_project? + source_repo.fork == true + end + + def number + raw_data.number + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_repo + raw_data.head.repo + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..b2bdbc10d7f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,7 +32,7 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 2d266ccfe9e..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -6,16 +6,20 @@ module Gitlab METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - def self.pool_size - current_application_settings[:metrics_pool_size] || 16 - end - - def self.timeout - current_application_settings[:metrics_timeout] || 10 + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 + } end def self.enabled? - current_application_settings[:metrics_enabled] || false + settings[:enabled] || false end def self.mri? @@ -26,32 +30,13 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= - (current_application_settings[:metrics_method_call_threshold] || 10) + @method_call_threshold ||= settings[:method_call_threshold] end def self.pool @pool end - def self.hostname - @hostname - end - - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) @@ -85,19 +70,15 @@ module Gitlab value.to_s.gsub('=', '\\=') end - @hostname = Socket.gethostname - # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? - @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = current_application_settings[:metrics_host] - user = current_application_settings[:metrics_username] - pw = current_application_settings[:metrics_password] - port = current_application_settings[:metrics_port] + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] InfluxDB::Client. - new(udp: { host: host, port: port }, username: user, password: pw) + new(udp: { host: host, port: port }) end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 06fc2f25948..d9fce2e6758 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -123,6 +123,8 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 753008df99a..7ea9555cc8c 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -17,11 +17,8 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash { - series: @series, - tags: @tags.merge( - hostname: Metrics.hostname, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' - ), + series: @series, + tags: @tags, values: @values, timestamp: @created_at.to_i * 1_000_000_000 } diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb deleted file mode 100644 index fe97d7a0534..00000000000 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Metrics - # Class for producing SQL queries with sensitive data stripped out. - class ObfuscatedSQL - REPLACEMENT = / - \d+(\.\d+)? # integers, floats - | '.+?' # single quoted strings - | \/.+?(?<!\\)\/ # regexps (including escaped slashes) - /x - - MYSQL_REPLACEMENTS = / - ".+?" # double quoted strings - /x - - # Regex to replace consecutive placeholders with a single one indicating - # the length. This can be useful when a "IN" statement uses thousands of - # IDs (storing this would just be a waste of space). - CONSECUTIVE = /(\?(\s*,\s*)?){2,}/ - - # sql - The raw SQL query as a String. - def initialize(sql) - @sql = sql - end - - # Returns a new, obfuscated SQL query. - def to_s - regex = REPLACEMENT - - if Gitlab::Database.mysql? - regex = Regexp.union(regex, MYSQL_REPLACEMENTS) - end - - sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match| - "#{match.count(',') + 1} values" - end - - # InfluxDB escapes double quotes upon output, so lets get rid of them - # whenever we can. - if Gitlab::Database.postgresql? - sql = sql.delete('"') - end - - sql.tr("\n", ' ') - end - end - end -end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -32,17 +32,15 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.add_tag(:request_method, env['REQUEST_METHOD']) - trans.add_tag(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_method, env['REQUEST_METHOD']) trans end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 998578e1c0a..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,9 +7,14 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) - @interval = interval - @metrics = [] + def initialize(interval = Metrics.settings[:sample_interval]) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -50,12 +55,11 @@ module Gitlab end def sample_memory_usage - @metrics << Metric.new('memory_usage', value: System.memory_usage) + add_metric('memory_usage', value: System.memory_usage) end def sample_file_descriptors - @metrics << Metric. - new('file_descriptors', value: System.file_descriptor_count) + add_metric('file_descriptors', value: System.file_descriptor_count) end if Metrics.mri? @@ -69,7 +73,7 @@ module Gitlab counts['Symbol'] = Symbol.all_symbols.length counts.each do |name, count| - @metrics << Metric.new('object_counts', { count: count }, type: name) + add_metric('object_counts', { count: count }, type: name) end end else @@ -91,7 +95,34 @@ module Gitlab stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - @metrics << Metric.new('gc_statistics', stats) + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7e0dcf99d92..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,6 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end @@ -32,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d947c128ce2..8008b3bc895 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -1,44 +1,18 @@ module Gitlab module Metrics module Subscribers - # Class for tracking raw SQL queries. - # - # Queries are obfuscated before being logged to ensure no private data is - # exposed via InfluxDB/Grafana. + # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber attach_to :active_record - SERIES = 'sql_queries' - def sql(event) return unless current_transaction - values = values_for(event) - tags = tags_for(event) - - current_transaction.add_metric(SERIES, values, tags) + current_transaction.increment(:sql_duration, event.duration) end private - def values_for(event) - { duration: event.duration } - end - - def tags_for(event) - sql = ObfuscatedSQL.new(event.payload[:sql]).to_s - tags = { sql: sql } - - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags - end - def current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index a61dbd989e7..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,45 +4,64 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - SERIES = 'transactions' + attr_reader :tags, :values - attr_reader :uuid, :tags + attr_accessor :action def self.current Thread.current[THREAD_KEY] end - # name - The name of this transaction as a String. - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] - @uuid = SecureRandom.uuid @started_at = nil @finished_at = nil - @tags = {} + @values = Hash.new(0) + @tags = {} + @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def increment(name, value) + @values[name] += value + end - @metrics << Metric.new(series, values, tags) + def set(name, value) + @values[name] = value end def add_tag(key, value) @@ -55,11 +74,29 @@ module Gitlab end def track_self - add_metric(SERIES, { duration: duration }, @tags) + values = { duration: duration, allocated_memory: allocated_memory } + + @values.each do |name, value| + values[name] = value + end + + add_metric('transactions', values, @tags) end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) + end + + def sidekiq? + Sidekiq.server? end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index be795649e59..4164e998dd1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -19,7 +19,7 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label merge_request snippet commit commit_range).each do |type| + %i(user label milestone merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do @references[type] ||= references(type, reference_context) end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 0469c5a61c3..2dc2953e328 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,7 +431,7 @@ namespace :gitlab do try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", - "find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" ) for_more_information( see_installation_guide_section "GitLab Shell" diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..8c63877e51c 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,8 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + namespace :gitlab do # Ask if the user wants to continue diff --git a/lib/version_check.rb b/lib/version_check.rb index ea23344948c..91ad07feee5 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -13,6 +13,6 @@ class VersionCheck end def host - 'https://version.gitlab.com/check.png' + 'https://version.gitlab.com/check.svg' end end |