From f60bceb988bd629f9adecc070ef5579d264f27c6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 13:20:06 +0100 Subject: Add CI data to projcet entity --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..e8154e0f383 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,9 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + + expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex + expose :runners_token end class ProjectMember < UserBasic -- cgit v1.2.1 From 0a21731e3bc400ceb9898c9efbc2a186f5348e09 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 15:00:25 +0100 Subject: Add ci fields in project create/update feature API --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..563c12e4f74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) + if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..7dd6b133f9b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,10 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -159,7 +162,10 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -215,7 +221,10 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? -- cgit v1.2.1 From ba9799b42f5bc861df1fc5d41c149cf72e9daf04 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 10:39:02 +0100 Subject: Update ./doc/api --- lib/api/projects.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dd6b133f9b..31b081266a8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -98,6 +98,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -146,6 +149,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -207,6 +213,9 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do -- cgit v1.2.1 From fc4108b38b5d0a7fc755c1977663f516b5dea0bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:27:20 +0100 Subject: Modify CI features in projects API --- lib/api/entities.rb | 4 +--- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 41 ++++++++++++++--------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8154e0f383..e3bc3316ce5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,9 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } - - expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex - expose :runners_token + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 563c12e4f74..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) + if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 31b081266a8..28c3583f351 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,9 +99,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -117,14 +115,12 @@ module API :namespace_id, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -149,9 +145,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -168,14 +161,12 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -194,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -213,9 +205,6 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do @@ -230,10 +219,7 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :visibility_level] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -247,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end -- cgit v1.2.1 From 41bcab118ae8be1baee0d750d99f46f584bd2fd4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:05:43 +0800 Subject: Fix #9963 reference_filter "Encoding::CompatibilityError" bug with some complex URL; --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 7198a8b03e2..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 932a247f5fb4a14b3e096ec88a73f8fce481eebb Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:32:25 +0800 Subject: Use CGI.escape instead of URI.escape, because URI is obsoleted. ref: https://github.com/ruby/ruby/commit/238b979f1789f95262a267d8df6239806f2859cc --- lib/gitlab/gitlab_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) -- cgit v1.2.1 From 5679ee0120ab45829b847d55d3a2253735856b3f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 14:59:30 +0100 Subject: Track memory allocated during a transaction This gives a very rough estimate of how much memory is allocated during a transaction. This only works reliably when using a single-threaded application server and a Ruby implementation with a GIL as otherwise memory allocated by other threads might skew the statistics. Sadly there's no way around this as Ruby doesn't provide a reliable way of gathering accurate object sizes upon allocation on a per-thread basis. --- lib/gitlab/metrics/transaction.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 86606b1c6d6..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -23,20 +23,29 @@ module Gitlab @values = Hash.new(0) @tags = {} @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end @@ -65,7 +74,7 @@ module Gitlab end def track_self - values = { duration: duration } + values = { duration: duration, allocated_memory: allocated_memory } @values.each do |name, value| values[name] = value -- cgit v1.2.1 From 355c341fe7c6b7c503386cf0b0e39cc02dbc8902 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 15:41:22 +0100 Subject: Stop tracking call stacks for instrumented views Where a vew is called from doesn't matter as much. We already know what action they belong to and this is more than enough information. By removing the file/line number from the list of tags we should also be able to reduce the number of series stored in InfluxDB. --- lib/gitlab/metrics.rb | 14 -------------- lib/gitlab/metrics/subscribers/action_view.rb | 10 +--------- 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 44356a0e42c..cdf7c168ff2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -36,20 +36,6 @@ module Gitlab @pool end - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -33,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction -- cgit v1.2.1 From a43fd5ce6d369c35b3ea421f045721d20f1c8ada Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 12 Jan 2016 21:34:23 -0500 Subject: Disable colorization if STDOUT is not a tty --- lib/tasks/gitlab/task_helpers.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..8c63877e51c 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,8 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From 23671600150cb022a09a77b8ea56a9465f19a013 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:29:48 +0100 Subject: Make the metrics sampler interval configurable --- lib/gitlab/metrics.rb | 3 ++- lib/gitlab/metrics/sampler.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cdf7c168ff2..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,7 +13,8 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port] + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 } end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 1ea425bc904..c2913841de3 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,7 +7,7 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) + def initialize(interval = Metrics.settings[:sample_interval]) @interval = interval @metrics = [] -- cgit v1.2.1 From 057eb824b5d7f38547506303dc80da6164715420 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:57:46 +0100 Subject: Randomize metrics sample intervals Sampling data at a fixed interval means we can potentially miss data from events occurring between sampling intervals. For example, say we sample data every 15 seconds but Unicorn workers get killed after 10 seconds. In this particular case it's possible to miss interesting data as the sampler will never get to actually submitting data. To work around this (at least for the most part) the sampling interval is randomized as following: 1. Take the user specified sampling interval (15 seconds by default) 2. Divide it by 2 (referred to as "half" below) 3. Generate a range (using a step of 0.1) from -"half" to "half" 4. Every time the sampler goes to sleep we'll grab the user provided interval and add a randomly chosen "adjustment" to it while making sure we don't pick the same value twice in a row. For a specified timeout of 15 this means the actual intervals can be anywhere between 7.5 and 22.5, but never can the same interval be used twice in a row. The rationale behind this change is that on dev.gitlab.org I'm sometimes seeing certain Gitlab::Git/Rugged objects being retained, but only for a few minutes every 24 hours. Knowing the code of Gitlab and how much memory it uses/leaks I suspect we're missing data due to workers getting terminated before the sampler can write its data to InfluxDB. --- lib/gitlab/metrics/sampler.rb | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index c2913841de3..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -8,8 +8,13 @@ module Gitlab class Sampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - @interval = interval - @metrics = [] + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -102,6 +107,23 @@ module Gitlab def sidekiq? Sidekiq.server? end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end end end end -- cgit v1.2.1