From f60bceb988bd629f9adecc070ef5579d264f27c6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 13:20:06 +0100 Subject: Add CI data to projcet entity --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..e8154e0f383 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,9 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + + expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex + expose :runners_token end class ProjectMember < UserBasic -- cgit v1.2.1 From 0a21731e3bc400ceb9898c9efbc2a186f5348e09 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 15:00:25 +0100 Subject: Add ci fields in project create/update feature API --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..563c12e4f74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) + if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..7dd6b133f9b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,10 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -159,7 +162,10 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -215,7 +221,10 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? -- cgit v1.2.1 From ba9799b42f5bc861df1fc5d41c149cf72e9daf04 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 10:39:02 +0100 Subject: Update ./doc/api --- lib/api/projects.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dd6b133f9b..31b081266a8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -98,6 +98,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -146,6 +149,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -207,6 +213,9 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do -- cgit v1.2.1 From fc4108b38b5d0a7fc755c1977663f516b5dea0bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:27:20 +0100 Subject: Modify CI features in projects API --- lib/api/entities.rb | 4 +--- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 41 ++++++++++++++--------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8154e0f383..e3bc3316ce5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,9 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } - - expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex - expose :runners_token + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 563c12e4f74..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) + if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 31b081266a8..28c3583f351 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,9 +99,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -117,14 +115,12 @@ module API :namespace_id, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -149,9 +145,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -168,14 +161,12 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -194,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -213,9 +205,6 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do @@ -230,10 +219,7 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :visibility_level] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -247,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end -- cgit v1.2.1 From 41bcab118ae8be1baee0d750d99f46f584bd2fd4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:05:43 +0800 Subject: Fix #9963 reference_filter "Encoding::CompatibilityError" bug with some complex URL; --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 7198a8b03e2..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 932a247f5fb4a14b3e096ec88a73f8fce481eebb Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:32:25 +0800 Subject: Use CGI.escape instead of URI.escape, because URI is obsoleted. ref: https://github.com/ruby/ruby/commit/238b979f1789f95262a267d8df6239806f2859cc --- lib/gitlab/gitlab_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) -- cgit v1.2.1 From 23671600150cb022a09a77b8ea56a9465f19a013 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:29:48 +0100 Subject: Make the metrics sampler interval configurable --- lib/gitlab/metrics.rb | 3 ++- lib/gitlab/metrics/sampler.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cdf7c168ff2..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,7 +13,8 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port] + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 } end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 1ea425bc904..c2913841de3 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,7 +7,7 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) + def initialize(interval = Metrics.settings[:sample_interval]) @interval = interval @metrics = [] -- cgit v1.2.1 From 057eb824b5d7f38547506303dc80da6164715420 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:57:46 +0100 Subject: Randomize metrics sample intervals Sampling data at a fixed interval means we can potentially miss data from events occurring between sampling intervals. For example, say we sample data every 15 seconds but Unicorn workers get killed after 10 seconds. In this particular case it's possible to miss interesting data as the sampler will never get to actually submitting data. To work around this (at least for the most part) the sampling interval is randomized as following: 1. Take the user specified sampling interval (15 seconds by default) 2. Divide it by 2 (referred to as "half" below) 3. Generate a range (using a step of 0.1) from -"half" to "half" 4. Every time the sampler goes to sleep we'll grab the user provided interval and add a randomly chosen "adjustment" to it while making sure we don't pick the same value twice in a row. For a specified timeout of 15 this means the actual intervals can be anywhere between 7.5 and 22.5, but never can the same interval be used twice in a row. The rationale behind this change is that on dev.gitlab.org I'm sometimes seeing certain Gitlab::Git/Rugged objects being retained, but only for a few minutes every 24 hours. Knowing the code of Gitlab and how much memory it uses/leaks I suspect we're missing data due to workers getting terminated before the sampler can write its data to InfluxDB. --- lib/gitlab/metrics/sampler.rb | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index c2913841de3..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -8,8 +8,13 @@ module Gitlab class Sampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - @interval = interval - @metrics = [] + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -102,6 +107,23 @@ module Gitlab def sidekiq? Sidekiq.server? end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end end end end -- cgit v1.2.1