From 4f0a38f1a833cab8c83e77a6c5d323057883188d Mon Sep 17 00:00:00 2001 From: Jeroen Nijhof Date: Wed, 21 Oct 2015 15:15:54 +0200 Subject: Added housekeeping for git repositories --- lib/gitlab/backend/shell.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 01b8bda05c6..59f7a45b791 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -149,6 +149,18 @@ module Gitlab "#{path}.git", tag_name]) end + # Gc repository + # + # path - project path with namespace + # + # Ex. + # gc("gitlab/gitlab-ci") + # + def gc(path) + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', + "#{path}.git"]) + end + # Add new key to gitlab-shell # # Ex. -- cgit v1.2.1 From 331154ffdf899a82c67487a70436ce49e494256b Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 24 Dec 2015 14:38:23 +0100 Subject: Escape reference link text --- lib/banzai/filter/abstract_reference_filter.rb | 17 ++++++++++------- lib/banzai/filter/reference_filter.rb | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index bdaa4721b4b..6b200dc2017 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -110,13 +110,7 @@ module Banzai url = matches[:url] if matches.names.include?("url") url ||= url_for_object(object, project) - text = link_text - unless text - text = object.reference_link_text(context[:project]) - - extras = object_link_text_extras(object, matches) - text += " (#{extras.join(", ")})" if extras.any? - end + text = link_text || escape_once(object_link_text(object, matches)) %( Date: Thu, 24 Dec 2015 14:43:07 +0100 Subject: Render milestone links as references --- lib/banzai/filter/abstract_reference_filter.rb | 38 ++++++++++++++----------- lib/banzai/filter/milestone_reference_filter.rb | 24 ++++++++++++++++ lib/banzai/pipeline/gfm_pipeline.rb | 1 + lib/gitlab/reference_extractor.rb | 2 +- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 lib/banzai/filter/milestone_reference_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 6b200dc2017..36d8c12e2b3 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -60,27 +60,31 @@ module Banzai end def call - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + if object_class.reference_pattern + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end - # `[Issue](#123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) + # `[Issue](#123)`, which is turned into + # `Issue` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end end - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `http://gitlab.example.com/namespace/project/issues/123` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + if object_class.link_reference_pattern + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `http://gitlab.example.com/namespace/project/issues/123` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `Issue` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb new file mode 100644 index 00000000000..f99202af5e8 --- /dev/null +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -0,0 +1,24 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces milestone references with links. + # + # This filter supports cross-project references. + class MilestoneReferenceFilter < AbstractReferenceFilter + def self.object_class + Milestone + end + + def find_object(project, id) + project.milestones.find_by(iid: id) + end + + def url_for_object(issue, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_milestone_url(project.namespace, project, milestone, + only_path: context[:only_path]) + end + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 38750b55ec7..838155e8831 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -22,6 +22,7 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, + Filter::MilestoneReferenceFilter, Filter::TaskListFilter ] diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 0a70d21b1ce..c87068051ab 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -18,7 +18,7 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label merge_request snippet commit commit_range).each do |type| + %i(user label milestone merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do @references[type] ||= references(type, project: project, current_user: current_user) end -- cgit v1.2.1 From 32543f3bd94b1b66dd949b0fb1f57bff3732eb45 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 24 Dec 2015 21:19:03 +0100 Subject: More escaping! --- lib/banzai/filter/abstract_reference_filter.rb | 8 ++++---- lib/banzai/filter/reference_filter.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 36d8c12e2b3..b99ccd98624 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -102,7 +102,7 @@ module Banzai project = project_from_ref(project_ref) if project && object = find_object(project, id) - title = escape_once(object_link_title(object)) + title = object_link_title(object) klass = reference_class(object_sym) data = data_attribute( @@ -114,11 +114,11 @@ module Banzai url = matches[:url] if matches.names.include?("url") url ||= url_for_object(object, project) - text = link_text || escape_once(object_link_text(object, matches)) + text = link_text || object_link_text(object, matches) %(#{text}) + title="#{escape_once(title)}" + class="#{klass}">#{escape_once(text)}) else match end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index c183702516a..a22a7a7afd3 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -44,7 +44,7 @@ module Banzai # Returns a String def data_attribute(attributes = {}) attributes[:reference_filter] = self.class.name.demodulize - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end def escape_once(html) -- cgit v1.2.1 From 141b8b67ff4cbe67778ff6815a51f49834e290b9 Mon Sep 17 00:00:00 2001 From: Michi302 Date: Mon, 28 Dec 2015 15:50:44 +0100 Subject: Make single user API endpoint return Entities::User instead of Entities::UserBasic --- lib/api/users.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 3400f0713ef..0d7813428e2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,7 +39,7 @@ module API if current_user.is_admin? present @user, with: Entities::UserFull else - present @user, with: Entities::UserBasic + present @user, with: Entities::User end end -- cgit v1.2.1 From d9d2e8a3e8de62beff685457e4e305f93122b206 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 29 Dec 2015 21:58:16 -0500 Subject: Support a single directory traversal in RelativeLinkFilter Prior, if we were viewing a blob at `https://example.com/namespace/project/blob/master/doc/some-file.md` and it contained a relative link such as `[README](../README.md)`, the resulting link when viewing the blob would be: `https://example.com/namespace/project/blob/README.md` which omits the `master` ref, resulting in a 404. --- lib/banzai/filter/relative_link_filter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5a081125f21..66f166939e4 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -91,7 +91,7 @@ module Banzai parts = request_path.split('/') parts.pop if path_type(request_path) != 'tree' - while parts.length > 1 && path.start_with?('../') + while path.start_with?('../') parts.pop path.sub!('../', '') end -- cgit v1.2.1 From 3077cb52d904154b98ee3e9aced5b3aadae86941 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 17:16:02 +0100 Subject: Use XPath for searching link nodes This is a tad faster than letting Nokogiri figure out whether it should evaluate the query as CSS or XPath and then actually evaluating it. --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 8ca05ace88c..7198a8b03e2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -124,7 +124,7 @@ module Banzai def replace_link_nodes_with_text(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -162,7 +162,7 @@ module Banzai def replace_link_nodes_with_href(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') -- cgit v1.2.1 From d3951dfaa13b9aaf11695ef10fa63456ac75cc48 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 17:16:59 +0100 Subject: Don't use delegate to delegate trivial methods Around 300 ms (in total) would be spent in these delegated methods due to the extra stuff ActiveSupport adds to the compiled methods. Because these delegations are so simple we can just manually define the methods, saving around 275 milliseconds. --- lib/banzai/filter/abstract_reference_filter.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 63ad8910c0f..230387c8383 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -47,7 +47,17 @@ module Banzai { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } end - delegate :object_class, :object_sym, :references_in, to: :class + def object_class + self.class.object_class + end + + def object_sym + self.class.object_sym + end + + def references_in(*args, &block) + self.class.references_in(*args, &block) + end def find_object(project, id) # Implement in child class -- cgit v1.2.1 From 054df415f94abe1e517a729e53cdb325d592d31b Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 18:16:53 +0100 Subject: Optimize CSS expressions produced by Nokogiri Nokogiri produces inefficient XPath expressions when given CSS expressions such as "a.gfm". Luckily these expressions can be optimized quite easily while still achieving the same results. In the two cases where this optimization is applied the run time has been reduced from around 170 ms to around 15 ms. --- lib/banzai/filter/redactor_filter.rb | 2 +- lib/banzai/filter/reference_gatherer_filter.rb | 2 +- lib/banzai/querying.rb | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 lib/banzai/querying.rb (limited to 'lib') diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index f01a32b5ae5..66f77902319 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -10,7 +10,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 12412ff7ea9..bef04112919 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -16,7 +16,7 @@ module Banzai end def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| gather_references(node) end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb new file mode 100644 index 00000000000..1e1b51e683e --- /dev/null +++ b/lib/banzai/querying.rb @@ -0,0 +1,18 @@ +module Banzai + module Querying + # Searches a Nokogiri document using a CSS query, optionally optimizing it + # whenever possible. + # + # document - A document/element to search. + # query - The CSS query to use. + # + # Returns a Nokogiri::XML::NodeSet. + def self.css(document, query) + # When using "a.foo" Nokogiri compiles this to "//a[...]" but + # "descendant::a[...]" is quite a bit faster and achieves the same result. + xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + + document.xpath(xpath) + end + end +end -- cgit v1.2.1 From a6c60127e3e2966b2f29fa6e6e79503b130c2b02 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:14:02 +0100 Subject: Removed tracking of raw SQL queries This particular setup had 3 problems: 1. Storing SQL queries as tags is very inefficient as InfluxDB ends up indexing every query (and they can get pretty large). Storing these as values instead means we can't always display the SQL as easily. 2. We already instrument ActiveRecord query methods, thus we already have timing information about database queries. 3. SQL obfuscation is difficult to get right and I'd rather not expose sensitive data by accident. --- lib/gitlab/metrics/obfuscated_sql.rb | 47 ------------------------ lib/gitlab/metrics/subscribers/active_record.rb | 48 ------------------------- 2 files changed, 95 deletions(-) delete mode 100644 lib/gitlab/metrics/obfuscated_sql.rb delete mode 100644 lib/gitlab/metrics/subscribers/active_record.rb (limited to 'lib') diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb deleted file mode 100644 index fe97d7a0534..00000000000 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Metrics - # Class for producing SQL queries with sensitive data stripped out. - class ObfuscatedSQL - REPLACEMENT = / - \d+(\.\d+)? # integers, floats - | '.+?' # single quoted strings - | \/.+?(? Date: Thu, 31 Dec 2015 17:47:07 +0100 Subject: Cache InfluxDB settings after the first use This ensures we don't need to load anything from either PostgreSQL or the Rails cache whenever creating new InfluxDB connections. --- lib/gitlab/metrics.rb | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 2d266ccfe9e..c5b98a0115e 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -6,16 +6,21 @@ module Gitlab METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - def self.pool_size - current_application_settings[:metrics_pool_size] || 16 - end - - def self.timeout - current_application_settings[:metrics_timeout] || 10 + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + username: current_application_settings[:metrics_username], + password: current_application_settings[:metrics_password], + port: current_application_settings[:metrics_port] + } end def self.enabled? - current_application_settings[:metrics_enabled] || false + settings[:enabled] || false end def self.mri? @@ -26,8 +31,7 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= - (current_application_settings[:metrics_method_call_threshold] || 10) + @method_call_threshold ||= settings[:method_call_threshold] end def self.pool @@ -90,11 +94,11 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? - @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = current_application_settings[:metrics_host] - user = current_application_settings[:metrics_username] - pw = current_application_settings[:metrics_password] - port = current_application_settings[:metrics_port] + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + user = settings[:username] + pw = settings[:password] + port = settings[:port] InfluxDB::Client. new(udp: { host: host, port: port }, username: user, password: pw) -- cgit v1.2.1 From bd9f86bb8abb4759a0c72f94fb0492b1ff8619b5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:51:12 +0100 Subject: Use separate series for Rails/Sidekiq transactions This removes the need for tagging all metrics with a "process_type" tag. --- lib/gitlab/metrics/metric.rb | 3 +-- lib/gitlab/metrics/rack_middleware.rb | 4 +++- lib/gitlab/metrics/sidekiq_middleware.rb | 4 +++- lib/gitlab/metrics/transaction.rb | 7 +++---- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 753008df99a..8319e628a40 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -19,8 +19,7 @@ module Gitlab { series: @series, tags: @tags.merge( - hostname: Metrics.hostname, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' + hostname: Metrics.hostname ), values: @values, timestamp: @created_at.to_i * 1_000_000_000 diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..bb9e4fcb918 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,6 +4,8 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' + SERIES = 'rails_transactions' + def initialize(app) @app = app end @@ -30,7 +32,7 @@ module Gitlab end def transaction_from_env(env) - trans = Transaction.new + trans = Transaction.new(SERIES) trans.add_tag(:request_method, env['REQUEST_METHOD']) trans.add_tag(:request_uri, env['REQUEST_URI']) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..6e804dd2562 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -4,8 +4,10 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware + SERIES = 'sidekiq_transactions' + def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new(SERIES) begin trans.run { yield } diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index a61dbd989e7..43a7dab5323 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,8 +4,6 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - SERIES = 'transactions' - attr_reader :uuid, :tags def self.current @@ -13,7 +11,8 @@ module Gitlab end # name - The name of this transaction as a String. - def initialize + def initialize(series) + @series = series @metrics = [] @uuid = SecureRandom.uuid @@ -55,7 +54,7 @@ module Gitlab end def track_self - add_metric(SERIES, { duration: duration }, @tags) + add_metric(@series, { duration: duration }, @tags) end def submit -- cgit v1.2.1 From cafc784ee1d5d0a0279077272af8ee435bb110e4 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:55:10 +0100 Subject: Removed tracking of hostnames for metrics This isn't hugely useful and mostly wastes InfluxDB space. We can re-add this whenever needed (but only once we really need it). --- lib/gitlab/metrics.rb | 6 ------ lib/gitlab/metrics/metric.rb | 6 ++---- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index c5b98a0115e..ee88ab34d6c 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -38,10 +38,6 @@ module Gitlab @pool end - def self.hostname - @hostname - end - # Returns a relative path and line number based on the last application call # frame. def self.last_relative_application_frame @@ -89,8 +85,6 @@ module Gitlab value.to_s.gsub('=', '\\=') end - @hostname = Socket.gethostname - # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 8319e628a40..7ea9555cc8c 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -17,10 +17,8 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash { - series: @series, - tags: @tags.merge( - hostname: Metrics.hostname - ), + series: @series, + tags: @tags, values: @values, timestamp: @created_at.to_i * 1_000_000_000 } -- cgit v1.2.1 From 1a1113f7c4ad6e5bceb898593cba6caecbdf0097 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 1 Jan 2016 22:11:34 -0500 Subject: Simplify `ContributionsCalendar#starting_year` and `#starting_month` --- lib/gitlab/contributions_calendar.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end -- cgit v1.2.1 From 086cfc8685a6489ca032899307c77f828f515fbb Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 26 Aug 2015 23:36:17 -0700 Subject: Fix API project lookups when querying with a namespace with dots Attempting to use the /projects/:id API by specifying :id in "namespace/project" format would always result in a 404 if the namespace contained a dot. The reason? From http://guides.rubyonrails.org/routing.html#specifying-constraints: "By default the :id parameter doesn't accept dots - this is because the dot is used as a separator for formatted routes. If you need to use a dot within an :id add a constraint which overrides this - for example id: /[^\/]+/ allows anything except a slash." Closes https://github.com/gitlabhq/gitlabhq/issues/9573 --- lib/api/projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a9e0960872a..0781236cf6d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,7 +3,7 @@ module API class Projects < Grape::API before { authenticate! } - resource :projects do + resource :projects, requirements: { id: /[^\/]+/ } do helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) -- cgit v1.2.1 From 96075be6f4688a59335130dc796132ad4f232442 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 11:37:46 +0100 Subject: Ability to increment custom transaction values This will be used to store/increment the total query/view rendering timings on a per transaction basis. This in turn can greatly reduce the amount of metrics stored. --- lib/gitlab/metrics/transaction.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 43a7dab5323..0aaebf262d4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -19,7 +19,8 @@ module Gitlab @started_at = nil @finished_at = nil - @tags = {} + @values = Hash.new(0) + @tags = {} end def duration @@ -44,6 +45,10 @@ module Gitlab @metrics << Metric.new(series, values, tags) end + def increment(name, value) + @values[name] += value + end + def add_tag(key, value) @tags[key] = value end @@ -54,7 +59,13 @@ module Gitlab end def track_self - add_metric(@series, { duration: duration }, @tags) + values = { duration: duration } + + @values.each do |name, value| + values[name] = value + end + + add_metric(@series, values, @tags) end def submit -- cgit v1.2.1 From 66a997a91403eef62ffd9fb789e899619d021a26 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:14:36 +0100 Subject: Track total query/view timings in transactions --- lib/gitlab/metrics/subscribers/action_view.rb | 1 + lib/gitlab/metrics/subscribers/active_record.rb | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lib/gitlab/metrics/subscribers/active_record.rb (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7e0dcf99d92..7c0105d543a 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,6 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb new file mode 100644 index 00000000000..8008b3bc895 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -0,0 +1,22 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total query duration of a transaction. + class ActiveRecord < ActiveSupport::Subscriber + attach_to :active_record + + def sql(event) + return unless current_transaction + + current_transaction.increment(:sql_duration, event.duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end -- cgit v1.2.1 From 825b46f8a3eb620f99192217d414b72dffe597d7 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:19:45 +0100 Subject: Track total method call times per transaction This makes it easier to see where time is spent without having to aggregate all the individual points in the method_calls series. --- lib/gitlab/metrics/instrumentation.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 06fc2f25948..d9fce2e6758 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -123,6 +123,8 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) -- cgit v1.2.1 From 2ea464bb272fa52ff34a188a921f0bc90811ca45 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:45:31 +0100 Subject: Use separate series for Rails/Sidekiq sample stats This removes the need for any tags to differentiate between Sidekiq and Rails statistics while still being able to separate the two. --- lib/gitlab/metrics/sampler.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 998578e1c0a..1ea425bc904 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -50,12 +50,11 @@ module Gitlab end def sample_memory_usage - @metrics << Metric.new('memory_usage', value: System.memory_usage) + add_metric('memory_usage', value: System.memory_usage) end def sample_file_descriptors - @metrics << Metric. - new('file_descriptors', value: System.file_descriptor_count) + add_metric('file_descriptors', value: System.file_descriptor_count) end if Metrics.mri? @@ -69,7 +68,7 @@ module Gitlab counts['Symbol'] = Symbol.all_symbols.length counts.each do |name, count| - @metrics << Metric.new('object_counts', { count: count }, type: name) + add_metric('object_counts', { count: count }, type: name) end end else @@ -91,7 +90,17 @@ module Gitlab stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - @metrics << Metric.new('gc_statistics', stats) + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? end end end -- cgit v1.2.1 From 2ee8f555996baca6b470d223ffad65419b730398 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 13:17:02 +0100 Subject: Automatically prefix transaction series names This ensures Rails and Sidekiq transactions are split into the series "rails_transactions" and "sidekiq_transactions" respectively. --- lib/gitlab/metrics/rack_middleware.rb | 4 +--- lib/gitlab/metrics/sidekiq_middleware.rb | 4 +--- lib/gitlab/metrics/transaction.rb | 15 +++++++++------ 3 files changed, 11 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index bb9e4fcb918..5c0587c4c51 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,8 +4,6 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' - SERIES = 'rails_transactions' - def initialize(app) @app = app end @@ -32,7 +30,7 @@ module Gitlab end def transaction_from_env(env) - trans = Transaction.new(SERIES) + trans = Transaction.new trans.add_tag(:request_method, env['REQUEST_METHOD']) trans.add_tag(:request_uri, env['REQUEST_URI']) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 6e804dd2562..ad441decfa2 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -4,10 +4,8 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware - SERIES = 'sidekiq_transactions' - def call(worker, message, queue) - trans = Transaction.new(SERIES) + trans = Transaction.new begin trans.run { yield } diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 0aaebf262d4..68b86de0655 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -10,9 +10,7 @@ module Gitlab Thread.current[THREAD_KEY] end - # name - The name of this transaction as a String. - def initialize(series) - @series = series + def initialize @metrics = [] @uuid = SecureRandom.uuid @@ -40,9 +38,10 @@ module Gitlab end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) + tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' - @metrics << Metric.new(series, values, tags) + @metrics << Metric.new("#{prefix}#{series}", values, tags) end def increment(name, value) @@ -65,12 +64,16 @@ module Gitlab values[name] = value end - add_metric(@series, values, @tags) + add_metric('transactions', values, @tags) end def submit Metrics.submit_metrics(@metrics.map(&:to_hash)) end + + def sidekiq? + Sidekiq.server? + end end end end -- cgit v1.2.1 From 4b027bc93a7875c3937f6b90ac1049b4a4d72da5 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 4 Jan 2016 14:29:06 +0100 Subject: Add DEBUG_BANZAI_CACHE env var to debug Banzai cache issue. --- lib/banzai/renderer.rb | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..910e1c6994e 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,13 +18,22 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + cacheless = cacheless_render(text, context) + + if cache_key && ENV["DEBUG_BANZAI_CACHE"] + cached = Rails.cache.fetch(cache_key) { cacheless } + + if cached != cacheless + Rails.logger.warn "Banzai cache mismatch" + Rails.logger.warn "Text: #{text.inspect}" + Rails.logger.warn "Context: #{context.inspect}" + Rails.logger.warn "Cache key: #{cache_key.inspect}" + Rails.logger.warn "Cacheless: #{cacheless.inspect}" + Rails.logger.warn "With cache: #{cached.inspect}" end - else - cacheless_render(text, context) end + + cacheless end def self.render_result(text, context = {}) -- cgit v1.2.1 From f60bceb988bd629f9adecc070ef5579d264f27c6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 13:20:06 +0100 Subject: Add CI data to projcet entity --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..e8154e0f383 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,9 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + + expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex + expose :runners_token end class ProjectMember < UserBasic -- cgit v1.2.1 From e57b506222f535774059cbb0f986621384c5a8f7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 5 Jan 2016 05:30:01 -0800 Subject: Suggest prefacing find command with sudo when base permissions are wrong Closes #5872 --- lib/tasks/gitlab/check.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 0469c5a61c3..2dc2953e328 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,7 +431,7 @@ namespace :gitlab do try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", - "find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" ) for_more_information( see_installation_guide_section "GitLab Shell" -- cgit v1.2.1 From 0a21731e3bc400ceb9898c9efbc2a186f5348e09 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 15:00:25 +0100 Subject: Add ci fields in project create/update feature API --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..563c12e4f74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) + if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..7dd6b133f9b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,10 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -159,7 +162,10 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -215,7 +221,10 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? -- cgit v1.2.1 From 097faeb481db2a4956b41049c041d55f5da4e2c1 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 5 Jan 2016 16:24:42 +0100 Subject: Get "Merge when build succeeds" to work when commits were pushed to MR target branch while builds were running --- lib/api/merge_requests.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3c1c6bda260..5c97fe1c88c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -211,7 +211,7 @@ module API unauthorized! unless merge_request.can_be_merged_by?(current_user) not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged if merge_request.unchecked? + merge_request.check_if_can_be_merged render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? -- cgit v1.2.1 From 0b661516324862506d5ec30c44cac704346a90ad Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 5 Jan 2016 16:45:53 +0100 Subject: Remove icon from milestone reference. --- lib/banzai/filter/milestone_reference_filter.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index f99202af5e8..e88b27c1fae 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -3,8 +3,6 @@ require 'banzai' module Banzai module Filter # HTML filter that replaces milestone references with links. - # - # This filter supports cross-project references. class MilestoneReferenceFilter < AbstractReferenceFilter def self.object_class Milestone -- cgit v1.2.1 From 95f1fe724a47b12b1d66a722be13d15c5fbd2688 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 16:52:45 -0200 Subject: Import GitHub Pull Requests into GitLab --- lib/gitlab/github_import/importer.rb | 64 ++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b5720f6e2cb..656fac9ac7e 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -12,7 +12,16 @@ module Gitlab end def execute - #Issues && Comments + import_issues + import_pull_requests + + true + end + + private + + def import_issues + # Issues && Comments client.list_issues(project.import_source, state: :all, sort: :created, direction: :asc).each do |issue| @@ -33,18 +42,59 @@ module Gitlab description: body, title: issue.title, state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) + author_id: gl_author_id(project, issue.user.id) ) end end end - private + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |pull_request| + body = @formatter.author_line(pull_request.user.login) + body += pull_request.body || "" + + source_branch = pull_request.head.ref + target_branch = pull_request.base.ref + + merge_request = MergeRequest.create!( + title: pull_request.title, + description: body, + source_project: project, + source_branch: source_branch, + target_project: project, + target_branch: target_branch, + state: merge_request_state(pull_request), + author_id: gl_author_id(project, pull_request.user.id), + assignee_id: gl_user_id(pull_request.assignee.try(:id)), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + end + end + + def merge_request_state(pull_request) + case true + when pull_request.state == 'closed' && pull_request.merged_at.present? + 'merged' + when pull_request.state == 'closed' + 'closed' + else + 'opened' + end + end + + def gl_author_id(project, github_id) + gl_user_id(github_id) || project.creator_id + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def gl_user_id(github_id) + if github_id + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end end end end -- cgit v1.2.1 From 223f7f53453d544a8c46c75ca9c54b8b60620961 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:08:54 -0200 Subject: Import comments on GitHub Pull Request into GitLab --- lib/gitlab/github_import/importer.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 656fac9ac7e..35a2930d4a1 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -71,9 +71,23 @@ module Gitlab created_at: pull_request.created_at, updated_at: pull_request.updated_at ) + + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end + def format_body(author, body) + @formatter.author_line(author) + (body || "") + end + def merge_request_state(pull_request) case true when pull_request.state == 'closed' && pull_request.merged_at.present? -- cgit v1.2.1 From f19bf0eaa73007081f2ab10f6a0fb176d5356e36 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:09:09 -0200 Subject: Import comments on the diff of a GitHub Pull Request into GitLab --- lib/gitlab/github_import/importer.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 35a2930d4a1..4b753d24601 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -81,6 +81,18 @@ module Gitlab updated_at: c.updated_at ) end + + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end @@ -99,6 +111,10 @@ module Gitlab end end + def generate_line_code(file_path, position) + Gitlab::Diff::LineCode.generate(file_path, position, 0) + end + def gl_author_id(project, github_id) gl_user_id(github_id) || project.creator_id end -- cgit v1.2.1 From d72b25811e3f0b722ae1c0906e2fe7dffd312403 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:42:56 -0200 Subject: Doesn't import GitHub PR where branches were no longer available --- lib/gitlab/github_import/importer.rb | 82 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 4b753d24601..f8a9e0d55ab 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -52,50 +52,56 @@ module Gitlab client.pull_requests(project.import_source, state: :all, sort: :created, direction: :asc).each do |pull_request| - body = @formatter.author_line(pull_request.user.login) - body += pull_request.body || "" - - source_branch = pull_request.head.ref - target_branch = pull_request.base.ref - - merge_request = MergeRequest.create!( - title: pull_request.title, - description: body, - source_project: project, - source_branch: source_branch, - target_project: project, - target_branch: target_branch, - state: merge_request_state(pull_request), - author_id: gl_author_id(project, pull_request.user.id), - assignee_id: gl_user_id(pull_request.assignee.try(:id)), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at + source_branch = find_branch(pull_request.head.ref) + target_branch = find_branch(pull_request.base.ref) + + if source_branch && target_branch + # Pull Request + merge_request = MergeRequest.create!( + title: pull_request.title, + description: format_body(pull_request.user.login, pull_request.body), + source_project: project, + source_branch: source_branch.name, + target_project: project, + target_branch: target_branch.name, + state: merge_request_state(pull_request), + author_id: gl_author_id(project, pull_request.user.id), + assignee_id: gl_user_id(pull_request.assignee.try(:id)), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at ) - end - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + # Comments on Pull Request + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + + # Comments on Pull Request diff + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end end + def find_branch(name) + project.repository.find_branch(name) + end + def format_body(author, body) @formatter.author_line(author) + (body || "") end -- cgit v1.2.1 From 6c846ef83d51a176002027e89245a4ea62b4f2bf Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 21:17:57 -0200 Subject: Extract methods to import comments on a GitHub Pull Request --- lib/gitlab/github_import/importer.rb | 52 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index f8a9e0d55ab..2c64f5cebc7 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -56,7 +56,6 @@ module Gitlab target_branch = find_branch(pull_request.base.ref) if source_branch && target_branch - # Pull Request merge_request = MergeRequest.create!( title: pull_request.title, description: format_body(pull_request.user.login, pull_request.body), @@ -71,33 +70,38 @@ module Gitlab updated_at: pull_request.updated_at ) - # Comments on Pull Request - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) - end - - # Comments on Pull Request diff - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) - end + import_comments_on_pull_request(merge_request, pull_request) + import_comments_on_pull_request_diff(merge_request, pull_request) end end end + def import_comments_on_pull_request(merge_request, pull_request) + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + end + + def import_comments_on_pull_request_diff(merge_request, pull_request) + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + end + def find_branch(name) project.repository.find_branch(name) end -- cgit v1.2.1 From dc72a8b30502dd28bf850c2dfdbf31b687fde5d3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 23 Dec 2015 15:04:46 -0200 Subject: Refactoring GithubImport::Importer --- lib/gitlab/github_import/comment.rb | 59 ++++++++++++++++++ lib/gitlab/github_import/importer.rb | 73 ++++------------------ lib/gitlab/github_import/pull_request.rb | 103 +++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 61 deletions(-) create mode 100644 lib/gitlab/github_import/comment.rb create mode 100644 lib/gitlab/github_import/pull_request.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/comment.rb b/lib/gitlab/github_import/comment.rb new file mode 100644 index 00000000000..55de78f889d --- /dev/null +++ b/lib/gitlab/github_import/comment.rb @@ -0,0 +1,59 @@ +module Gitlab + module GithubImport + class Comment + attr_reader :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + @formatter.author_line(author) + body + end + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2c64f5cebc7..7c495655012 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -51,80 +51,31 @@ module Gitlab def import_pull_requests client.pull_requests(project.import_source, state: :all, sort: :created, - direction: :asc).each do |pull_request| - source_branch = find_branch(pull_request.head.ref) - target_branch = find_branch(pull_request.base.ref) + direction: :asc).each do |raw_data| + pull_request = PullRequest.new(project, raw_data) - if source_branch && target_branch - merge_request = MergeRequest.create!( - title: pull_request.title, - description: format_body(pull_request.user.login, pull_request.body), - source_project: project, - source_branch: source_branch.name, - target_project: project, - target_branch: target_branch.name, - state: merge_request_state(pull_request), - author_id: gl_author_id(project, pull_request.user.id), - assignee_id: gl_user_id(pull_request.assignee.try(:id)), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - import_comments_on_pull_request(merge_request, pull_request) - import_comments_on_pull_request_diff(merge_request, pull_request) + if pull_request.valid? + merge_request = MergeRequest.create!(pull_request.attributes) + import_comments_on_pull_request(merge_request, raw_data) + import_comments_on_pull_request_diff(merge_request, raw_data) end end end def import_comments_on_pull_request(merge_request, pull_request) - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + client.issue_comments(project.import_source, pull_request.number).each do |raw_data| + comment = Comment.new(project, raw_data) + merge_request.notes.create!(comment.attributes) end end def import_comments_on_pull_request_diff(merge_request, pull_request) - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + client.pull_request_comments(project.import_source, pull_request.number).each do |raw_data| + comment = Comment.new(project, raw_data) + merge_request.notes.create!(comment.attributes) end end - def find_branch(name) - project.repository.find_branch(name) - end - - def format_body(author, body) - @formatter.author_line(author) + (body || "") - end - - def merge_request_state(pull_request) - case true - when pull_request.state == 'closed' && pull_request.merged_at.present? - 'merged' - when pull_request.state == 'closed' - 'closed' - else - 'opened' - end - end - - def generate_line_code(file_path, position) - Gitlab::Diff::LineCode.generate(file_path, position, 0) - end - def gl_author_id(project, github_id) gl_user_id(github_id) || project.creator_id end diff --git a/lib/gitlab/github_import/pull_request.rb b/lib/gitlab/github_import/pull_request.rb new file mode 100644 index 00000000000..61e846472f2 --- /dev/null +++ b/lib/gitlab/github_import/pull_request.rb @@ -0,0 +1,103 @@ +module Gitlab + module GithubImport + class PullRequest + attr_reader :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def source_project + project + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end -- cgit v1.2.1 From 98909dd12cd27b85921962326bcaf651c092dcd5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 23 Dec 2015 20:02:59 -0200 Subject: Generate separate comments when importing GitHub Issues into GitLab --- lib/gitlab/github_import/base_formatter.rb | 21 +++++ lib/gitlab/github_import/comment.rb | 59 ------------ lib/gitlab/github_import/comment_formatter.rb | 45 +++++++++ lib/gitlab/github_import/importer.rb | 59 ++++-------- lib/gitlab/github_import/issue_formatter.rb | 66 +++++++++++++ lib/gitlab/github_import/pull_request.rb | 103 --------------------- lib/gitlab/github_import/pull_request_formatter.rb | 93 +++++++++++++++++++ 7 files changed, 244 insertions(+), 202 deletions(-) create mode 100644 lib/gitlab/github_import/base_formatter.rb delete mode 100644 lib/gitlab/github_import/comment.rb create mode 100644 lib/gitlab/github_import/comment_formatter.rb create mode 100644 lib/gitlab/github_import/issue_formatter.rb delete mode 100644 lib/gitlab/github_import/pull_request.rb create mode 100644 lib/gitlab/github_import/pull_request_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/comment.rb b/lib/gitlab/github_import/comment.rb deleted file mode 100644 index 55de78f889d..00000000000 --- a/lib/gitlab/github_import/comment.rb +++ /dev/null @@ -1,59 +0,0 @@ -module Gitlab - module GithubImport - class Comment - attr_reader :project, :raw_data - - def initialize(project, raw_data) - @project = project - @raw_data = raw_data - @formatter = Gitlab::ImportFormatter.new - end - - def attributes - { - project: project, - note: note, - commit_id: raw_data.commit_id, - line_code: line_code, - author_id: author_id, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at - } - end - - private - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def line_code - if on_diff? - Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) - end - end - - def on_diff? - raw_data.path && raw_data.position - end - - def note - @formatter.author_line(author) + body - end - - def gl_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) - end - end - end -end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 7c495655012..38ca7372202 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -21,29 +21,17 @@ module Gitlab private def import_issues - # Issues && Comments client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - body = @formatter.author_line(issue.user.login) - body += issue.body || "" + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - if issue.comments > 0 - body += @formatter.comments_header - - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end - - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_author_id(project, issue.user.id) - ) end end end @@ -52,39 +40,30 @@ module Gitlab client.pull_requests(project.import_source, state: :all, sort: :created, direction: :asc).each do |raw_data| - pull_request = PullRequest.new(project, raw_data) + pull_request = PullRequestFormatter.new(project, raw_data) if pull_request.valid? merge_request = MergeRequest.create!(pull_request.attributes) - import_comments_on_pull_request(merge_request, raw_data) - import_comments_on_pull_request_diff(merge_request, raw_data) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) end end end - def import_comments_on_pull_request(merge_request, pull_request) - client.issue_comments(project.import_source, pull_request.number).each do |raw_data| - comment = Comment.new(project, raw_data) - merge_request.notes.create!(comment.attributes) - end - end - - def import_comments_on_pull_request_diff(merge_request, pull_request) - client.pull_request_comments(project.import_source, pull_request.number).each do |raw_data| - comment = Comment.new(project, raw_data) - merge_request.notes.create!(comment.attributes) - end + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) end - def gl_author_id(project, github_id) - gl_user_id(github_id) || project.creator_id + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) end - def gl_user_id(github_id) - if github_id - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/pull_request.rb b/lib/gitlab/github_import/pull_request.rb deleted file mode 100644 index 61e846472f2..00000000000 --- a/lib/gitlab/github_import/pull_request.rb +++ /dev/null @@ -1,103 +0,0 @@ -module Gitlab - module GithubImport - class PullRequest - attr_reader :project, :raw_data - - def initialize(project, raw_data) - @project = project - @raw_data = raw_data - @formatter = Gitlab::ImportFormatter.new - end - - def attributes - { - title: raw_data.title, - description: description, - source_project: source_project, - source_branch: source_branch.name, - target_project: target_project, - target_branch: target_branch.name, - state: state, - author_id: author_id, - assignee_id: assignee_id, - created_at: raw_data.created_at, - updated_at: updated_at - } - end - - def valid? - source_branch.present? && target_branch.present? - end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gl_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - @formatter.author_line(author) + body - end - - def source_project - project - end - - def source_branch - source_project.repository.find_branch(raw_data.head.ref) - end - - def target_project - project - end - - def target_branch - target_project.repository.find_branch(raw_data.base.ref) - end - - def state - @state ||= case true - when raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - when raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at - else - raw_data.updated_at - end - end - - def gl_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) - end - end - end -end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..42dc09c2ac5 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,93 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def number + raw_data.number + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end -- cgit v1.2.1 From 837a9065f0ff192d2efd55edcc2658a92c127b21 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 5 Jan 2016 15:15:36 -0200 Subject: Ensure that we're only importing local pull requests --- lib/gitlab/github_import/importer.rb | 2 +- lib/gitlab/github_import/pull_request_formatter.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 38ca7372202..2b0afbc7b39 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -42,7 +42,7 @@ module Gitlab direction: :asc).each do |raw_data| pull_request = PullRequestFormatter.new(project, raw_data) - if pull_request.valid? + if !pull_request.cross_project? && pull_request.valid? merge_request = MergeRequest.create!(pull_request.attributes) import_comments(pull_request.number, merge_request) import_comments_on_diff(pull_request.number, merge_request) diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 42dc09c2ac5..b7c47958cc7 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -17,6 +17,10 @@ module Gitlab } end + def cross_project? + source_repo.fork == true + end + def number raw_data.number end @@ -57,6 +61,10 @@ module Gitlab project end + def source_repo + raw_data.head.repo + end + def source_branch source_project.repository.find_branch(raw_data.head.ref) end -- cgit v1.2.1 From 045e8cc38c608b46924ce1ef4de3e8c2adefb1ba Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 29 Dec 2015 20:23:07 -0500 Subject: Update version check images to use SVG --- lib/version_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/version_check.rb b/lib/version_check.rb index ea23344948c..91ad07feee5 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -13,6 +13,6 @@ class VersionCheck end def host - 'https://version.gitlab.com/check.png' + 'https://version.gitlab.com/check.svg' end end -- cgit v1.2.1 From 1e6fc0c6a440ad707d990282ab7a93c178e35cfa Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 30 Dec 2015 16:44:22 -0500 Subject: Define a limited set of filters for SingleLinePipeline Removes the following filters from its parent GfmPipeline: - SyntaxHighlightFilter - UploadLinkFilter - TableOfContentsFilter - LabelReferenceFilter - TaskListFilter Closes #1697 --- lib/banzai/pipeline/single_line_pipeline.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 6725c9039a9..a3c9d4f43aa 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,7 +3,23 @@ require 'banzai' module Banzai module Pipeline class SingleLinePipeline < GfmPipeline + def self.filters + @filters ||= [ + Filter::SanitizationFilter, + Filter::EmojiFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter, + + Filter::UserReferenceFilter, + Filter::IssueReferenceFilter, + Filter::ExternalIssueReferenceFilter, + Filter::MergeRequestReferenceFilter, + Filter::SnippetReferenceFilter, + Filter::CommitRangeReferenceFilter, + Filter::CommitReferenceFilter, + ] + end end end end -- cgit v1.2.1 From b9ed3961b55cf3dbc1a6d4c841d295dd23161c90 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 6 Jan 2016 13:25:13 +0100 Subject: Revert "Add DEBUG_BANZAI_CACHE env var to debug Banzai cache issue." This reverts commit 4b027bc93a7875c3937f6b90ac1049b4a4d72da5. --- lib/banzai/renderer.rb | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 910e1c6994e..115ae914524 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,5 +1,7 @@ module Banzai module Renderer + CACHE_ENABLED = false + # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -18,22 +20,13 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - cacheless = cacheless_render(text, context) - - if cache_key && ENV["DEBUG_BANZAI_CACHE"] - cached = Rails.cache.fetch(cache_key) { cacheless } - - if cached != cacheless - Rails.logger.warn "Banzai cache mismatch" - Rails.logger.warn "Text: #{text.inspect}" - Rails.logger.warn "Context: #{context.inspect}" - Rails.logger.warn "Cache key: #{cache_key.inspect}" - Rails.logger.warn "Cacheless: #{cacheless.inspect}" - Rails.logger.warn "With cache: #{cached.inspect}" + if cache_key && CACHE_ENABLED + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) end + else + cacheless_render(text, context) end - - cacheless end def self.render_result(text, context = {}) -- cgit v1.2.1 From cf19efec3ac0ab4510359dd71df3d511762230c3 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 6 Jan 2016 13:26:02 +0100 Subject: Revert "Temporarily disable Markdown caching" This reverts commit d337d5e7137d9b3fd0f9b8890a3ba9296323acc7. --- lib/banzai/renderer.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..891c0fd7749 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,7 +18,7 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED + if cache_key Rails.cache.fetch(cache_key) do cacheless_render(text, context) end -- cgit v1.2.1 From 7549102bb727daecc51da84af39956b32fc41537 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 6 Jan 2016 15:30:02 +0100 Subject: Store SQL/view timings in milliseconds Transaction timings are also already stored in milliseconds, this keeps things consistent. --- lib/gitlab/metrics/subscribers/action_view.rb | 8 ++++++-- lib/gitlab/metrics/subscribers/active_record.rb | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..84d9e383625 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,7 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) - current_transaction.increment(:view_duration, event.duration) + current_transaction.increment(:view_duration, duration(event)) current_transaction.add_metric(SERIES, values, tags) end @@ -28,7 +28,7 @@ module Gitlab end def values_for(event) - { duration: event.duration } + { duration: duration(event) } end def tags_for(event) @@ -48,6 +48,10 @@ module Gitlab def current_transaction Transaction.current end + + def duration(event) + event.duration * 1000.0 + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 8008b3bc895..6fa73e7a3be 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return unless current_transaction - current_transaction.increment(:sql_duration, event.duration) + current_transaction.increment(:sql_duration, duration(event)) end private @@ -16,6 +16,10 @@ module Gitlab def current_transaction Transaction.current end + + def duration(event) + event.duration * 1000.0 + end end end end -- cgit v1.2.1 From 8fdc00bd4c59183a20a60a6b93228268230bbd2e Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 6 Jan 2016 17:49:02 +0100 Subject: Remove InfluxDB username/password InfluxDB over UDP doesn't use authentication, thus there's no need for these settings. --- lib/gitlab/metrics.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index ee88ab34d6c..44356a0e42c 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,8 +13,6 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - username: current_application_settings[:metrics_username], - password: current_application_settings[:metrics_password], port: current_application_settings[:metrics_port] } end @@ -90,12 +88,10 @@ module Gitlab if enabled? @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do host = settings[:host] - user = settings[:username] - pw = settings[:password] port = settings[:port] InfluxDB::Client. - new(udp: { host: host, port: port }, username: user, password: pw) + new(udp: { host: host, port: port }) end end end -- cgit v1.2.1 From ba9799b42f5bc861df1fc5d41c149cf72e9daf04 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 10:39:02 +0100 Subject: Update ./doc/api --- lib/api/projects.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dd6b133f9b..31b081266a8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -98,6 +98,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -146,6 +149,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -207,6 +213,9 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do -- cgit v1.2.1 From 7ed3a5a240e4997b24d11b96e27126dfaa575abe Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 11:47:06 +0100 Subject: Revert "Store SQL/view timings in milliseconds" This reverts commit 7549102bb727daecc51da84af39956b32fc41537. Apparently I was wrong about ActiveSupport::Notifications::Event#duration returning the duration in seconds, instead it returns it in milliseconds already. --- lib/gitlab/metrics/subscribers/action_view.rb | 8 ++------ lib/gitlab/metrics/subscribers/active_record.rb | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 84d9e383625..7c0105d543a 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,7 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) - current_transaction.increment(:view_duration, duration(event)) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end @@ -28,7 +28,7 @@ module Gitlab end def values_for(event) - { duration: duration(event) } + { duration: event.duration } end def tags_for(event) @@ -48,10 +48,6 @@ module Gitlab def current_transaction Transaction.current end - - def duration(event) - event.duration * 1000.0 - end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 6fa73e7a3be..8008b3bc895 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return unless current_transaction - current_transaction.increment(:sql_duration, duration(event)) + current_transaction.increment(:sql_duration, event.duration) end private @@ -16,10 +16,6 @@ module Gitlab def current_transaction Transaction.current end - - def duration(event) - event.duration * 1000.0 - end end end end -- cgit v1.2.1 From 364b07cff0183956ea11962b94c70448767351d3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 12:44:15 +0100 Subject: Removed UUIDs from metrics transactions While useful for finding out what methods/views belong to a transaction this might result in too much data being stored in InfluxDB. --- lib/gitlab/metrics/transaction.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 68b86de0655..306656d30fe 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :uuid, :tags + attr_reader :tags def self.current Thread.current[THREAD_KEY] @@ -12,7 +12,6 @@ module Gitlab def initialize @metrics = [] - @uuid = SecureRandom.uuid @started_at = nil @finished_at = nil @@ -38,7 +37,6 @@ module Gitlab end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) prefix = sidekiq? ? 'sidekiq_' : 'rails_' @metrics << Metric.new("#{prefix}#{series}", values, tags) -- cgit v1.2.1 From 7b10cb6f0f5758c17dd950587982ff400d7aa971 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 13:05:00 +0100 Subject: Store request methods/URIs as values Since filtering by these values is very rare (they're mostly just displayed as-is) we don't need to waste any index space by saving them as tags. By storing them as values we also greatly reduce the number of series in InfluxDB. --- lib/gitlab/metrics/rack_middleware.rb | 4 ++-- lib/gitlab/metrics/transaction.rb | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..e7a2f26d48b 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -32,8 +32,8 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.add_tag(:request_method, env['REQUEST_METHOD']) - trans.add_tag(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_method, env['REQUEST_METHOD']) trans end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 306656d30fe..73131cc6ef2 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags + attr_reader :tags, :values def self.current Thread.current[THREAD_KEY] @@ -46,6 +46,10 @@ module Gitlab @values[name] += value end + def set(name, value) + @values[name] = value + end + def add_tag(key, value) @tags[key] = value end -- cgit v1.2.1 From 1886d727f738895bb552151d59d4024f405522e2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 13:37:14 +0100 Subject: Add API project upload endpoint --- lib/api/projects.rb | 12 +++++++++++- lib/gitlab/email/receiver.rb | 7 ++----- lib/gitlab/fogbugz_import/importer.rb | 4 +--- 3 files changed, 14 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..8b1390e3289 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -269,7 +269,7 @@ module API # Remove a forked_from relationship # # Parameters: - # id: (required) - The ID of the project being marked as a fork + # id: (required) - The ID of the project being marked as a fork # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do @@ -278,6 +278,16 @@ module API user_project.forked_project_link.destroy end end + + # Upload a file + # + # Parameters: + # id: (required) - The ID of the project + # file: (required) - The file to be uploaded + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + # search for projects current_user has access to # # Parameters: diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2b252b32887..2ca21af5bc8 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -74,7 +74,7 @@ module Gitlab def sent_notification return nil unless reply_key - + SentNotification.for(reply_key) end @@ -82,10 +82,7 @@ module Gitlab attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) attachments.each do |link| - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - reply << "\n\n#{text}" + reply << "\n\n#{link[:markdown]}" end reply diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 403ebeec474..d5f755f90e5 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,9 +232,7 @@ module Gitlab return nil if res.nil? - text = "[#{res['alt']}](#{res['url']})" - text = "!#{text}" if res['is_image'] - text + text = res['markdown'] end def build_attachment_url(rel_url) -- cgit v1.2.1 From 706d7eb0b7a1d54604a11e1202e44069c6acccee Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 13:37:59 +0100 Subject: Satisfy Rubocp --- lib/gitlab/fogbugz_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index d5f755f90e5..0e6bee732f1 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,7 +232,7 @@ module Gitlab return nil if res.nil? - text = res['markdown'] + res['markdown'] end def build_attachment_url(rel_url) -- cgit v1.2.1 From c3865bda0257506e3e9ba8352913eb288f0e3e34 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 14:04:01 +0100 Subject: Properly set task-list class on single item task lists --- lib/banzai/filter/task_list_filter.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index bdf7c2ebdfc..d0ce13003a5 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -12,13 +12,18 @@ module Banzai # # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class(node, *new_class_names) + def add_css_class_with_fix(node, *new_class_names) if new_class_names.include?('task-list') - super if node.children.any? { |c| c['class'] == 'task-list-item' } - else - super + # Don't add class to all lists + return + elsif new_class_names.include?('task-list-item') + add_css_class_without_fix(node.parent, 'task-list') end + + add_css_class_without_fix(node, *new_class_names) end + + alias_method_chain :add_css_class, :fix end end end -- cgit v1.2.1 From 69209612e1793fcebcdb784074056d7a02b0f6f7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 22 Dec 2015 12:15:06 -0800 Subject: Suppress e-mails on failed builds if allow_failure is set Every time I push to GitLab, I get > 2 emails saying a spec failed when I don't care about benchmarks and other specs that have `allow_failure` set to `true`. --- lib/gitlab/build_data_builder.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 86bfa0a4378..34e949130da 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -23,6 +23,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_allow_failure: build.allow_failure, # TODO: do we still need it? project_id: project.id, -- cgit v1.2.1 From 4c90ed52fe6ff7b1155088c46460c411e121feb3 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Fri, 8 Jan 2016 10:10:04 +0100 Subject: Delete tag via API --- lib/api/tags.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'lib') diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 47621f443e6..2d8a9e51bb9 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -40,6 +40,27 @@ module API end end + # Delete tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # DELETE /projects/:id/repository/tags/:tag + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + authorize_push_project + result = DeleteTagService.new(user_project, current_user). + execute(params[:tag_name]) + + if result[:status] == :success + { + tag_name: params[:tag_name] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + # Add release notes to tag # # Parameters: -- cgit v1.2.1 From 8d7a968d6dbc70d31953bac5262c86c4ca9dc7d9 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 8 Jan 2016 14:31:39 +0100 Subject: Handle missing DBs in connect_to_db? This ensures CurrentSettings#connect_to_db? returns "false" in the event of a database not existing, instead of raising an error. --- lib/gitlab/current_settings.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7a86c09158e..7f938780ab1 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -41,6 +41,9 @@ module Gitlab use_db && ActiveRecord::Base.connection.active? && !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') + + rescue ActiveRecord::NoDatabaseError + false end end end -- cgit v1.2.1 From 0614793b38db4711053cbcb4fa80d9c8cc492eec Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 8 Jan 2016 17:38:53 +0100 Subject: DRY up upload and download services --- lib/gitlab/fogbugz_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 0e6bee732f1..db580b5e578 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,7 +232,7 @@ module Gitlab return nil if res.nil? - res['markdown'] + res[:markdown] end def build_attachment_url(rel_url) -- cgit v1.2.1 From 67aa0b8c4cbf762211ad178efb537f1649d91776 Mon Sep 17 00:00:00 2001 From: Drew Blessing Date: Thu, 31 Dec 2015 13:22:51 -0600 Subject: Optimize LDAP and add a search timeout --- lib/gitlab/ldap/access.rb | 8 ++++++-- lib/gitlab/ldap/adapter.rb | 24 +++++++++++++++--------- lib/gitlab/ldap/config.rb | 4 ++++ 3 files changed, 25 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..b2bdbc10d7f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,7 +32,7 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap -- cgit v1.2.1 From fc4108b38b5d0a7fc755c1977663f516b5dea0bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:27:20 +0100 Subject: Modify CI features in projects API --- lib/api/entities.rb | 4 +--- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 41 ++++++++++++++--------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8154e0f383..e3bc3316ce5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,9 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } - - expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex - expose :runners_token + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 563c12e4f74..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) + if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 31b081266a8..28c3583f351 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,9 +99,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -117,14 +115,12 @@ module API :namespace_id, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -149,9 +145,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -168,14 +161,12 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -194,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -213,9 +205,6 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do @@ -230,10 +219,7 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :visibility_level] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -247,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end -- cgit v1.2.1 From 35b501f30ae9e121151ad6a2140d036e5ef3b0f5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 11 Jan 2016 16:51:01 +0100 Subject: Tag all transaction metrics with an "action" tag Without this it's impossible to find out what methods/views/queries are executed by a certain controller or Sidekiq worker. While this will increase the total number of series it should stay within reasonable limits due to the amount of "actions" being small enough. --- lib/gitlab/metrics/rack_middleware.rb | 6 ++---- lib/gitlab/metrics/sidekiq_middleware.rb | 7 +------ lib/gitlab/metrics/transaction.rb | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e7a2f26d48b..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -39,10 +39,8 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 73131cc6ef2..86606b1c6d6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -6,11 +6,15 @@ module Gitlab attr_reader :tags, :values + attr_accessor :action + def self.current Thread.current[THREAD_KEY] end - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] @started_at = nil @@ -18,6 +22,7 @@ module Gitlab @values = Hash.new(0) @tags = {} + @action = action end def duration @@ -70,7 +75,15 @@ module Gitlab end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) end def sidekiq? -- cgit v1.2.1 From 41bcab118ae8be1baee0d750d99f46f584bd2fd4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:05:43 +0800 Subject: Fix #9963 reference_filter "Encoding::CompatibilityError" bug with some complex URL; --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 7198a8b03e2..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 932a247f5fb4a14b3e096ec88a73f8fce481eebb Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:32:25 +0800 Subject: Use CGI.escape instead of URI.escape, because URI is obsoleted. ref: https://github.com/ruby/ruby/commit/238b979f1789f95262a267d8df6239806f2859cc --- lib/gitlab/gitlab_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) -- cgit v1.2.1 From 5679ee0120ab45829b847d55d3a2253735856b3f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 14:59:30 +0100 Subject: Track memory allocated during a transaction This gives a very rough estimate of how much memory is allocated during a transaction. This only works reliably when using a single-threaded application server and a Ruby implementation with a GIL as otherwise memory allocated by other threads might skew the statistics. Sadly there's no way around this as Ruby doesn't provide a reliable way of gathering accurate object sizes upon allocation on a per-thread basis. --- lib/gitlab/metrics/transaction.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 86606b1c6d6..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -23,20 +23,29 @@ module Gitlab @values = Hash.new(0) @tags = {} @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end @@ -65,7 +74,7 @@ module Gitlab end def track_self - values = { duration: duration } + values = { duration: duration, allocated_memory: allocated_memory } @values.each do |name, value| values[name] = value -- cgit v1.2.1 From 355c341fe7c6b7c503386cf0b0e39cc02dbc8902 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 15:41:22 +0100 Subject: Stop tracking call stacks for instrumented views Where a vew is called from doesn't matter as much. We already know what action they belong to and this is more than enough information. By removing the file/line number from the list of tags we should also be able to reduce the number of series stored in InfluxDB. --- lib/gitlab/metrics.rb | 14 -------------- lib/gitlab/metrics/subscribers/action_view.rb | 10 +--------- 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 44356a0e42c..cdf7c168ff2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -36,20 +36,6 @@ module Gitlab @pool end - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -33,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction -- cgit v1.2.1 From a43fd5ce6d369c35b3ea421f045721d20f1c8ada Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 12 Jan 2016 21:34:23 -0500 Subject: Disable colorization if STDOUT is not a tty --- lib/tasks/gitlab/task_helpers.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..8c63877e51c 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,8 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From 23671600150cb022a09a77b8ea56a9465f19a013 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:29:48 +0100 Subject: Make the metrics sampler interval configurable --- lib/gitlab/metrics.rb | 3 ++- lib/gitlab/metrics/sampler.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cdf7c168ff2..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,7 +13,8 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port] + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 } end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 1ea425bc904..c2913841de3 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,7 +7,7 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) + def initialize(interval = Metrics.settings[:sample_interval]) @interval = interval @metrics = [] -- cgit v1.2.1 From 057eb824b5d7f38547506303dc80da6164715420 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:57:46 +0100 Subject: Randomize metrics sample intervals Sampling data at a fixed interval means we can potentially miss data from events occurring between sampling intervals. For example, say we sample data every 15 seconds but Unicorn workers get killed after 10 seconds. In this particular case it's possible to miss interesting data as the sampler will never get to actually submitting data. To work around this (at least for the most part) the sampling interval is randomized as following: 1. Take the user specified sampling interval (15 seconds by default) 2. Divide it by 2 (referred to as "half" below) 3. Generate a range (using a step of 0.1) from -"half" to "half" 4. Every time the sampler goes to sleep we'll grab the user provided interval and add a randomly chosen "adjustment" to it while making sure we don't pick the same value twice in a row. For a specified timeout of 15 this means the actual intervals can be anywhere between 7.5 and 22.5, but never can the same interval be used twice in a row. The rationale behind this change is that on dev.gitlab.org I'm sometimes seeing certain Gitlab::Git/Rugged objects being retained, but only for a few minutes every 24 hours. Knowing the code of Gitlab and how much memory it uses/leaks I suspect we're missing data due to workers getting terminated before the sampler can write its data to InfluxDB. --- lib/gitlab/metrics/sampler.rb | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index c2913841de3..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -8,8 +8,13 @@ module Gitlab class Sampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - @interval = interval - @metrics = [] + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -102,6 +107,23 @@ module Gitlab def sidekiq? Sidekiq.server? end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end end end end -- cgit v1.2.1