From 4f0a38f1a833cab8c83e77a6c5d323057883188d Mon Sep 17 00:00:00 2001 From: Jeroen Nijhof Date: Wed, 21 Oct 2015 15:15:54 +0200 Subject: Added housekeeping for git repositories --- lib/gitlab/backend/shell.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 01b8bda05c6..59f7a45b791 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -149,6 +149,18 @@ module Gitlab "#{path}.git", tag_name]) end + # Gc repository + # + # path - project path with namespace + # + # Ex. + # gc("gitlab/gitlab-ci") + # + def gc(path) + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', + "#{path}.git"]) + end + # Add new key to gitlab-shell # # Ex. -- cgit v1.2.1 From 331154ffdf899a82c67487a70436ce49e494256b Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 24 Dec 2015 14:38:23 +0100 Subject: Escape reference link text --- lib/banzai/filter/abstract_reference_filter.rb | 17 ++++++++++------- lib/banzai/filter/reference_filter.rb | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index bdaa4721b4b..6b200dc2017 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -110,13 +110,7 @@ module Banzai url = matches[:url] if matches.names.include?("url") url ||= url_for_object(object, project) - text = link_text - unless text - text = object.reference_link_text(context[:project]) - - extras = object_link_text_extras(object, matches) - text += " (#{extras.join(", ")})" if extras.any? - end + text = link_text || escape_once(object_link_text(object, matches)) %( Date: Thu, 24 Dec 2015 14:43:07 +0100 Subject: Render milestone links as references --- lib/banzai/filter/abstract_reference_filter.rb | 38 ++++++++++++++----------- lib/banzai/filter/milestone_reference_filter.rb | 24 ++++++++++++++++ lib/banzai/pipeline/gfm_pipeline.rb | 1 + lib/gitlab/reference_extractor.rb | 2 +- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 lib/banzai/filter/milestone_reference_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 6b200dc2017..36d8c12e2b3 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -60,27 +60,31 @@ module Banzai end def call - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + if object_class.reference_pattern + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end - # `[Issue](#123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) + # `[Issue](#123)`, which is turned into + # `Issue` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end end - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `http://gitlab.example.com/namespace/project/issues/123` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + if object_class.link_reference_pattern + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `http://gitlab.example.com/namespace/project/issues/123` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `Issue` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb new file mode 100644 index 00000000000..f99202af5e8 --- /dev/null +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -0,0 +1,24 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces milestone references with links. + # + # This filter supports cross-project references. + class MilestoneReferenceFilter < AbstractReferenceFilter + def self.object_class + Milestone + end + + def find_object(project, id) + project.milestones.find_by(iid: id) + end + + def url_for_object(issue, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_milestone_url(project.namespace, project, milestone, + only_path: context[:only_path]) + end + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 38750b55ec7..838155e8831 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -22,6 +22,7 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, + Filter::MilestoneReferenceFilter, Filter::TaskListFilter ] diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 0a70d21b1ce..c87068051ab 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -18,7 +18,7 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label merge_request snippet commit commit_range).each do |type| + %i(user label milestone merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do @references[type] ||= references(type, project: project, current_user: current_user) end -- cgit v1.2.1 From f4e7a8893d71bcbc92a5ecd16c95593f17d1ddf5 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 24 Dec 2015 19:05:57 +0100 Subject: Add builds API (listing, showing trace) --- lib/api/api.rb | 2 ++ lib/api/builds.rb | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/api/entities.rb | 13 +++++++++ 3 files changed, 95 insertions(+) create mode 100644 lib/api/builds.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7834262d612..266b5f48f8f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,5 +54,7 @@ module API mount Keys mount Tags mount Triggers + + mount Builds end end diff --git a/lib/api/builds.rb b/lib/api/builds.rb new file mode 100644 index 00000000000..ce4892b5eeb --- /dev/null +++ b/lib/api/builds.rb @@ -0,0 +1,80 @@ +module API + # Projects builds API + class Builds < Grape::API + before { authenticate! } + + resource :projects do + # Get a project repository commits + # + # Parameters: + # id (required) - The ID of a project + # scope (optional) - The scope of builds to show (one of: all, finished, running) + # page (optional) - The page number for pagination (default: 1) + # per_page (ooptional) - The value of items per page to show (default 30) + # Example Request: + # GET /projects/:id/builds/all + get ':id/builds' do + all_builds = user_project.builds + builds = all_builds.order('created_at DESC') + builds = + case params[:scope] + when 'all' + builds + when 'finished' + builds.finished + when 'running' + builds.running + when 'pending' + builds.pending + when 'success' + builds.success + when 'failed' + builds.failed + else + builds.running_or_pending.reverse_order + end + + page = (params[:page] || 1).to_i + per_page = (params[:per_page] || 30).to_i + + present builds.page(page).per(per_page), with: Entities::Build + end + + # Get a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/builds/:build_id + get ':id/builds/:build_id' do + present get_build(params[:build_id]), with: Entities::Build + end + + # Get a trace of a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/build/:build_id/trace + get ':id/builds/:build_id/trace' do + trace = get_build(params[:build_id]).trace + trace = + unless trace.nil? + trace.split("\n") + else + [] + end + + present trace + end + end + + helpers do + def get_build(id) + user_project.builds.where(id: id).first + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f8511ac5f5c..0bf50490eac 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -366,5 +366,18 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Build < Grape::Entity + expose :id + expose :status + expose :stage + expose :name + expose :ref + expose :commit + expose :runner + expose :created_at + expose :started_at + expose :finished_at + end end end -- cgit v1.2.1 From b5fef34f1e3beb60e7184cfb3420976bfa367137 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 24 Dec 2015 19:18:01 +0100 Subject: Fix example request url --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index ce4892b5eeb..0ddb9e98de6 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -12,7 +12,7 @@ module API # page (optional) - The page number for pagination (default: 1) # per_page (ooptional) - The value of items per page to show (default 30) # Example Request: - # GET /projects/:id/builds/all + # GET /projects/:id/builds get ':id/builds' do all_builds = user_project.builds builds = all_builds.order('created_at DESC') -- cgit v1.2.1 From 32543f3bd94b1b66dd949b0fb1f57bff3732eb45 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 24 Dec 2015 21:19:03 +0100 Subject: More escaping! --- lib/banzai/filter/abstract_reference_filter.rb | 8 ++++---- lib/banzai/filter/reference_filter.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 36d8c12e2b3..b99ccd98624 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -102,7 +102,7 @@ module Banzai project = project_from_ref(project_ref) if project && object = find_object(project, id) - title = escape_once(object_link_title(object)) + title = object_link_title(object) klass = reference_class(object_sym) data = data_attribute( @@ -114,11 +114,11 @@ module Banzai url = matches[:url] if matches.names.include?("url") url ||= url_for_object(object, project) - text = link_text || escape_once(object_link_text(object, matches)) + text = link_text || object_link_text(object, matches) %(#{text}) + title="#{escape_once(title)}" + class="#{klass}">#{escape_once(text)}) else match end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index c183702516a..a22a7a7afd3 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -44,7 +44,7 @@ module Banzai # Returns a String def data_attribute(attributes = {}) attributes[:reference_filter] = self.class.name.demodulize - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end def escape_once(html) -- cgit v1.2.1 From 83d42c1518274dc0af0f49fda3a10e846569cbcc Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Fri, 25 Dec 2015 18:13:55 +0200 Subject: Revert upvotes and downvotes params to MR API --- lib/api/entities.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f8511ac5f5c..26e7c956e8f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -166,7 +166,6 @@ module API class MergeRequest < ProjectEntity expose :target_branch, :source_branch - # deprecated, always returns 0 expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id -- cgit v1.2.1 From f39959d00a1358ba7d73ebeaccb827738c8151ba Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 13:09:51 +0100 Subject: Add some fixes to builds API --- lib/api/builds.rb | 27 +++++++++------------------ lib/api/entities.rb | 20 ++++++++++++++++++-- 2 files changed, 27 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 0ddb9e98de6..863be0d5e40 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -15,23 +15,15 @@ module API # GET /projects/:id/builds get ':id/builds' do all_builds = user_project.builds - builds = all_builds.order('created_at DESC') + builds = all_builds.order('id DESC') builds = case params[:scope] - when 'all' - builds when 'finished' builds.finished when 'running' builds.running - when 'pending' - builds.pending - when 'success' - builds.success - when 'failed' - builds.failed else - builds.running_or_pending.reverse_order + builds end page = (params[:page] || 1).to_i @@ -59,15 +51,14 @@ module API # Example Request: # GET /projects/:id/build/:build_id/trace get ':id/builds/:build_id/trace' do - trace = get_build(params[:build_id]).trace - trace = - unless trace.nil? - trace.split("\n") - else - [] - end + build = get_build(params[:build_id]) + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary - present trace + trace = build.trace + body trace end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0bf50490eac..76b5d14f202 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,14 +367,30 @@ module API expose :id, :variables end + class CiCommit < Grape::Entity + expose :id + expose :ref + expose :sha + expose :committed_at + end + + class CiRunner < Grape::Entity + expose :id + expose :token + expose :description + expose :active + expose :is_shared + expose :name + end + class Build < Grape::Entity expose :id expose :status expose :stage expose :name expose :ref - expose :commit - expose :runner + expose :commit, with: CiCommit + expose :runner, with: CiRunner expose :created_at expose :started_at expose :finished_at -- cgit v1.2.1 From f4cff4dcd0b11d4597efe454731838fbf5803516 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 13:33:03 +0100 Subject: Modify build pagination to use 'paginate' helper --- lib/api/builds.rb | 5 +---- lib/api/helpers.rb | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 863be0d5e40..f219b0f5241 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -26,10 +26,7 @@ module API builds end - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 30).to_i - - present builds.page(page).per(per_page), with: Entities::Build + present paginate(builds), with: Entities::Build end # Get a specific build of a project diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..8fb5cd6ab63 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,8 +97,10 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) + page = (params[:page] || 1).to_i + per_page = (params[:per_page] || 30).to_i + + paginated = relation.page(page).per(per_page) add_pagination_headers(paginated, per_page) paginated -- cgit v1.2.1 From d398e78ea09c1f88800cd4c99f92dd8e5d1d0d39 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 15:49:13 +0100 Subject: Add endpoint for getting builds for a specific commit --- lib/api/builds.rb | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index f219b0f5241..7488863cdcf 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -4,7 +4,7 @@ module API before { authenticate! } resource :projects do - # Get a project repository commits + # Get a project builds # # Parameters: # id (required) - The ID of a project @@ -14,18 +14,21 @@ module API # Example Request: # GET /projects/:id/builds get ':id/builds' do - all_builds = user_project.builds - builds = all_builds.order('id DESC') - builds = - case params[:scope] - when 'finished' - builds.finished - when 'running' - builds.running - else - builds - end + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + present paginate(builds), with: Entities::Build + end + # GET builds for a specific commit of a project + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The SHA id of a commit + # Example Request: + # GET /projects/:id/builds/commit/:sha + get ':id/builds/commit/:sha' do + builds = user_project.ci_commits.find_by_sha(params[:sha]).builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build end @@ -63,6 +66,17 @@ module API def get_build(id) user_project.builds.where(id: id).first end + + def filter_builds(builds, scope) + case scope + when 'finished' + builds.finished + when 'running' + builds.running + else + builds + end + end end end end -- cgit v1.2.1 From 141b8b67ff4cbe67778ff6815a51f49834e290b9 Mon Sep 17 00:00:00 2001 From: Michi302 Date: Mon, 28 Dec 2015 15:50:44 +0100 Subject: Make single user API endpoint return Entities::User instead of Entities::UserBasic --- lib/api/users.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 3400f0713ef..0d7813428e2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,7 +39,7 @@ module API if current_user.is_admin? present @user, with: Entities::UserFull else - present @user, with: Entities::UserBasic + present @user, with: Entities::User end end -- cgit v1.2.1 From e7d0746d9319119c459581615ae4205139bee444 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 16:38:02 +0100 Subject: Add 'not_found' notifications in build/trace details --- lib/api/builds.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 7488863cdcf..5ac24d0367a 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -19,7 +19,7 @@ module API present paginate(builds), with: Entities::Build end - # GET builds for a specific commit of a project + # Get builds for a specific commit of a project # # Parameters: # id (required) - The ID of a project @@ -40,7 +40,10 @@ module API # Example Request: # GET /projects/:id/builds/:build_id get ':id/builds/:build_id' do - present get_build(params[:build_id]), with: Entities::Build + build = get_build(params[:build_id]) + return not_found!(build) unless build + + present build, with: Entities::Build end # Get a trace of a specific build of a project @@ -52,6 +55,7 @@ module API # GET /projects/:id/build/:build_id/trace get ':id/builds/:build_id/trace' do build = get_build(params[:build_id]) + return not_found!(build) unless build header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' -- cgit v1.2.1 From 8d4555037a844d21dbf56c2995cff30782af920b Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 16:38:29 +0100 Subject: Add cancel/retry endpoints to build API --- lib/api/builds.rb | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 5ac24d0367a..16e4549d280 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -64,6 +64,42 @@ module API trace = build.trace body trace end + + # cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/cancel + post ':id/builds/:build_id/cancel' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + build.cancel + + present build, with: Entities::Build + end + + # cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/retry + post ':id/builds/:build_id/retry' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.retryable? + + build = Ci::Build.retry(build) + + present build, with: Entities::Build + end end helpers do @@ -81,6 +117,10 @@ module API builds end end + + def authorize_manage_builds! + authorize! :manage_builds, user_project + end end end end -- cgit v1.2.1 From b2fbeb377d8ba17352d40817b0b444196cb97636 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 17:01:02 +0100 Subject: Remove changes in 'paginate' helper --- lib/api/helpers.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8fb5cd6ab63..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,10 +97,8 @@ module API end def paginate(relation) - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 30).to_i - - paginated = relation.page(page).per(per_page) + per_page = params[:per_page].to_i + paginated = relation.page(params[:page]).per(per_page) add_pagination_headers(paginated, per_page) paginated -- cgit v1.2.1 From a3469d914aaf28a1184247cbe72e5197ce7ca006 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 28 Dec 2015 18:21:34 -0200 Subject: reCAPTCHA is configurable through Admin Settings, no reload needed. --- lib/gitlab/recaptcha.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lib/gitlab/recaptcha.rb (limited to 'lib') diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb new file mode 100644 index 00000000000..70e7f25d518 --- /dev/null +++ b/lib/gitlab/recaptcha.rb @@ -0,0 +1,14 @@ +module Gitlab + module Recaptcha + def self.load_configurations! + if current_application_settings.recaptcha_enabled + ::Recaptcha.configure do |config| + config.public_key = current_application_settings.recaptcha_site_key + config.private_key = current_application_settings.recaptcha_private_key + end + + true + end + end + end +end -- cgit v1.2.1 From 03478e6d5b98a723fbb349dac2c8495f75909a08 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 29 Dec 2015 13:40:08 +0100 Subject: Strip newlines from obfuscated SQL Newlines aren't really needed and they may mess with InfluxDB's line protocol. --- lib/gitlab/metrics/obfuscated_sql.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb index 481aca56efb..2e932fb3049 100644 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ b/lib/gitlab/metrics/obfuscated_sql.rb @@ -40,7 +40,7 @@ module Gitlab sql = sql.delete('"') end - sql + sql.gsub("\n", ' ') end end end -- cgit v1.2.1 From 620e7bb3d60c3685b494b26e256b793a47621da4 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 29 Dec 2015 13:40:42 +0100 Subject: Write to InfluxDB directly via UDP This removes the need for Sidekiq and any overhead/problems introduced by TCP. There are a few things to take into account: 1. When writing data to InfluxDB you may still get an error if the server becomes unavailable during the write. Because of this we're catching all exceptions and just ignore them (for now). 2. Writing via UDP apparently requires the timestamp to be in nanoseconds. Without this data either isn't written properly. 3. Due to the restrictions on UDP buffer sizes we're writing metrics one by one, instead of writing all of them at once. --- lib/gitlab/metrics.rb | 38 ++++++++++++++++++++++++++++++-- lib/gitlab/metrics/metric.rb | 2 +- lib/gitlab/metrics/obfuscated_sql.rb | 2 +- lib/gitlab/metrics/sampler.rb | 2 +- lib/gitlab/metrics/sidekiq_middleware.rb | 7 ------ lib/gitlab/metrics/transaction.rb | 2 +- 6 files changed, 40 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 9470633b065..0869d007ca5 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -66,6 +66,39 @@ module Gitlab end end + def self.submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool.with do |connection| + prepared.each do |metric| + begin + connection.write_points([metric]) + rescue StandardError + end + end + end + end + + def self.prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def self.escape_value(value) + value.to_s.gsub('=', '\\=') + end + @hostname = Socket.gethostname # When enabled this should be set before being used as the usual pattern @@ -73,11 +106,12 @@ module Gitlab if enabled? @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do host = settings[:metrics_host] - db = settings[:metrics_database] user = settings[:metrics_username] pw = settings[:metrics_password] + port = settings[:metrics_port] - InfluxDB::Client.new(db, host: host, username: user, password: pw) + InfluxDB::Client. + new(udp: { host: host, port: port }, username: user, password: pw) end end end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f592f4e571f..79241f56874 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -26,7 +26,7 @@ module Gitlab process_type: Sidekiq.server? ? 'sidekiq' : 'rails' ), values: @values, - timestamp: @created_at.to_i + timestamp: @created_at.to_i * 1_000_000_000 } end end diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb index 2e932fb3049..fe97d7a0534 100644 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ b/lib/gitlab/metrics/obfuscated_sql.rb @@ -40,7 +40,7 @@ module Gitlab sql = sql.delete('"') end - sql.gsub("\n", ' ') + sql.tr("\n", ' ') end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 828ee1f8c62..998578e1c0a 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -46,7 +46,7 @@ module Gitlab end def flush - MetricsWorker.perform_async(@metrics.map(&:to_hash)) + Metrics.submit_metrics(@metrics.map(&:to_hash)) end def sample_memory_usage diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ec10707d1fb..ad441decfa2 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,13 +5,6 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - # We don't want to track the MetricsWorker itself as otherwise we'll end - # up in an infinite loop. - if worker.class == MetricsWorker - yield - return - end - trans = Transaction.new begin diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 568f9d6ae0c..a61dbd989e7 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -59,7 +59,7 @@ module Gitlab end def submit - MetricsWorker.perform_async(@metrics.map(&:to_hash)) + Metrics.submit_metrics(@metrics.map(&:to_hash)) end end end -- cgit v1.2.1 From 701e5de9108faa65a118e8822560d39fd8ea9996 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 29 Dec 2015 15:49:12 +0100 Subject: Use Gitlab::CurrentSettings for InfluxDB This ensures we can still start up even when not connecting to a database. --- lib/gitlab/metrics.rb | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 0869d007ca5..2d266ccfe9e 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,36 +1,21 @@ module Gitlab module Metrics + extend Gitlab::CurrentSettings + RAILS_ROOT = Rails.root.to_s METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - # Returns the current settings, ensuring we _always_ have a default set of - # metrics settings (even during tests, when the migrations are lacking, - # etc). This ensures the application is able to boot up even when the - # migrations have not been executed. - def self.settings - if ApplicationSetting.table_exists? and curr = ApplicationSetting.current - curr - else - { - metrics_pool_size: 16, - metrics_timeout: 10, - metrics_enabled: false, - metrics_method_call_threshold: 10 - } - end - end - def self.pool_size - settings[:metrics_pool_size] + current_application_settings[:metrics_pool_size] || 16 end def self.timeout - settings[:metrics_timeout] + current_application_settings[:metrics_timeout] || 10 end def self.enabled? - settings[:metrics_enabled] + current_application_settings[:metrics_enabled] || false end def self.mri? @@ -41,7 +26,8 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= settings[:metrics_method_call_threshold] + @method_call_threshold ||= + (current_application_settings[:metrics_method_call_threshold] || 10) end def self.pool @@ -105,10 +91,10 @@ module Gitlab # "@foo ||= bar" is _not_ thread-safe. if enabled? @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = settings[:metrics_host] - user = settings[:metrics_username] - pw = settings[:metrics_password] - port = settings[:metrics_port] + host = current_application_settings[:metrics_host] + user = current_application_settings[:metrics_username] + pw = current_application_settings[:metrics_password] + port = current_application_settings[:metrics_port] InfluxDB::Client. new(udp: { host: host, port: port }, username: user, password: pw) -- cgit v1.2.1 From d2601211a0c7a44666501dea82a8488b08f8faa7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 29 Dec 2015 23:12:36 +0100 Subject: Add specs for build listings in API --- lib/api/builds.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 16e4549d280..a519224d2b8 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -27,7 +27,10 @@ module API # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do - builds = user_project.ci_commits.find_by_sha(params[:sha]).builds.order('id DESC') + commit = user_project.ci_commits.find_by_sha(params[:sha]) + return not_found! unless commit + + builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build end @@ -65,7 +68,7 @@ module API body trace end - # cancel a specific build of a project + # Cancel a specific build of a project # # parameters: # id (required) - the id of a project @@ -83,7 +86,7 @@ module API present build, with: Entities::Build end - # cancel a specific build of a project + # Retry a specific build of a project # # parameters: # id (required) - the id of a project -- cgit v1.2.1 From ff98c631c1004247656677568989e5ed68fc88f3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 30 Dec 2015 14:26:54 -0200 Subject: Make sure that is no pending migrations in Gitlab::CurrentSettings --- lib/gitlab/current_settings.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 46a4ef0e31f..7a86c09158e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -38,7 +38,9 @@ module Gitlab true end - use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings') + use_db && ActiveRecord::Base.connection.active? && + !ActiveRecord::Migrator.needs_migration? && + ActiveRecord::Base.connection.table_exists?('application_settings') end end end -- cgit v1.2.1 From d9d2e8a3e8de62beff685457e4e305f93122b206 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 29 Dec 2015 21:58:16 -0500 Subject: Support a single directory traversal in RelativeLinkFilter Prior, if we were viewing a blob at `https://example.com/namespace/project/blob/master/doc/some-file.md` and it contained a relative link such as `[README](../README.md)`, the resulting link when viewing the blob would be: `https://example.com/namespace/project/blob/README.md` which omits the `master` ref, resulting in a 404. --- lib/banzai/filter/relative_link_filter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5a081125f21..66f166939e4 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -91,7 +91,7 @@ module Banzai parts = request_path.split('/') parts.pop if path_type(request_path) != 'tree' - while parts.length > 1 && path.start_with?('../') + while path.start_with?('../') parts.pop path.sub!('../', '') end -- cgit v1.2.1 From c936e4e3c8282df17f2dc89b305233b7608003ca Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 15:53:52 +0100 Subject: Removed various default metrics tags While it's useful to keep track of the different versions (Ruby, GitLab, etc) doing so for every point wastes disk space and possibly also RAM (which InfluxDB is all to eager to gobble up). If we want to see the performance differences between different GitLab versions simply looking at the performance since the last release date should suffice. --- lib/gitlab/metrics/metric.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 79241f56874..753008df99a 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -19,11 +19,8 @@ module Gitlab { series: @series, tags: @tags.merge( - hostname: Metrics.hostname, - ruby_engine: RUBY_ENGINE, - ruby_version: RUBY_VERSION, - gitlab_version: Gitlab::VERSION, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' + hostname: Metrics.hostname, + process_type: Sidekiq.server? ? 'sidekiq' : 'rails' ), values: @values, timestamp: @created_at.to_i * 1_000_000_000 -- cgit v1.2.1 From 3077cb52d904154b98ee3e9aced5b3aadae86941 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 17:16:02 +0100 Subject: Use XPath for searching link nodes This is a tad faster than letting Nokogiri figure out whether it should evaluate the query as CSS or XPath and then actually evaluating it. --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 8ca05ace88c..7198a8b03e2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -124,7 +124,7 @@ module Banzai def replace_link_nodes_with_text(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -162,7 +162,7 @@ module Banzai def replace_link_nodes_with_href(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') -- cgit v1.2.1 From d3951dfaa13b9aaf11695ef10fa63456ac75cc48 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 17:16:59 +0100 Subject: Don't use delegate to delegate trivial methods Around 300 ms (in total) would be spent in these delegated methods due to the extra stuff ActiveSupport adds to the compiled methods. Because these delegations are so simple we can just manually define the methods, saving around 275 milliseconds. --- lib/banzai/filter/abstract_reference_filter.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 63ad8910c0f..230387c8383 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -47,7 +47,17 @@ module Banzai { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } end - delegate :object_class, :object_sym, :references_in, to: :class + def object_class + self.class.object_class + end + + def object_sym + self.class.object_sym + end + + def references_in(*args, &block) + self.class.references_in(*args, &block) + end def find_object(project, id) # Implement in child class -- cgit v1.2.1 From 054df415f94abe1e517a729e53cdb325d592d31b Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Dec 2015 18:16:53 +0100 Subject: Optimize CSS expressions produced by Nokogiri Nokogiri produces inefficient XPath expressions when given CSS expressions such as "a.gfm". Luckily these expressions can be optimized quite easily while still achieving the same results. In the two cases where this optimization is applied the run time has been reduced from around 170 ms to around 15 ms. --- lib/banzai/filter/redactor_filter.rb | 2 +- lib/banzai/filter/reference_gatherer_filter.rb | 2 +- lib/banzai/querying.rb | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 lib/banzai/querying.rb (limited to 'lib') diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index f01a32b5ae5..66f77902319 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -10,7 +10,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 12412ff7ea9..bef04112919 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -16,7 +16,7 @@ module Banzai end def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| gather_references(node) end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb new file mode 100644 index 00000000000..1e1b51e683e --- /dev/null +++ b/lib/banzai/querying.rb @@ -0,0 +1,18 @@ +module Banzai + module Querying + # Searches a Nokogiri document using a CSS query, optionally optimizing it + # whenever possible. + # + # document - A document/element to search. + # query - The CSS query to use. + # + # Returns a Nokogiri::XML::NodeSet. + def self.css(document, query) + # When using "a.foo" Nokogiri compiles this to "//a[...]" but + # "descendant::a[...]" is quite a bit faster and achieves the same result. + xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + + document.xpath(xpath) + end + end +end -- cgit v1.2.1 From ea4777ff501e370a39ae30e76a955136afe3c1fa Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 15:19:13 +0100 Subject: Add features for list and show details of variables in API --- lib/api/api.rb | 2 ++ lib/api/entities.rb | 4 ++++ lib/api/variables.rb | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 lib/api/variables.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7834262d612..a9e1913f0f2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,5 +54,7 @@ module API mount Keys mount Tags mount Triggers + + mount Variables end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..f71d072f269 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -365,5 +365,9 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Variable < Grape::Entity + expose :id, :key, :value + end end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb new file mode 100644 index 00000000000..6517150f6f4 --- /dev/null +++ b/lib/api/variables.rb @@ -0,0 +1,43 @@ +module API + # Projects variables API + class Variables < Grape::API + before { authenticate! } + before { authorize_admin_project } + + resource :projects do + # Get project variables + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/variables + get ':id/variables' do + variables = user_project.variables + present paginate(variables), with: Entities::Variable + end + + # Get specifica bariable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated + # as ID other ways it's treated as `key` + # Example Reuest: + # GET /projects/:id/variables/:variable_id + get ':id/variables/:variable_id' do + variable_id = params[:variable_id] + variables = user_project.variables + variables = + if variable_id.match(/^\d+$/) + variables.where(id: variable_id.to_i) + else + variables.where(key: variable_id) + end + + present variables.first, with: Entities::Variable + end + end + end +end -- cgit v1.2.1 From a692ce1c079703c4f3947e1d0a29547189e94d0f Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 16:25:49 +0100 Subject: Add update feature for variables API --- lib/api/variables.rb | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 6517150f6f4..6522ecba70c 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -24,7 +24,7 @@ module API # id (required) - The ID of a project # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated # as ID other ways it's treated as `key` - # Example Reuest: + # Example Request: # GET /projects/:id/variables/:variable_id get ':id/variables/:variable_id' do variable_id = params[:variable_id] @@ -38,6 +38,25 @@ module API present variables.first, with: Entities::Variable end + + # Update existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID of a variable + # key (optional) - new value for `key` field of variable + # value (optional) - new value for `value` field of variable + # Example Request: + # PUT /projects/:id/variables/:variable_id + put ':id/variables/:variable_id' do + variable = user_project.variables.where(id: params[:variable_id].to_i).first + + variable.key = params[:key] + variable.value = params[:value] + variable.save! + + present variable, with: Entities::Variable + end end end end -- cgit v1.2.1 From 0d014feb1d216e692882976f0d70c3227eaec4ca Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 16:56:03 +0100 Subject: Add delete feature to variables API --- lib/api/variables.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 6522ecba70c..c70c7cd9d7b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -57,6 +57,18 @@ module API present variable, with: Entities::Variable end + + # Delete existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID of a variable + # Exanoke Reqyest: + # DELETE /projects/:id/variables/:variable_id + delete ':id/variables/:variable_id' do + variable = user_project.variables.where(id: params[:variable_id].to_i).first + variable.destroy + end end end end -- cgit v1.2.1 From c5177dd5e2171b047a695802c979cf779522ba8a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 17:03:11 +0100 Subject: Add missing 'not_found' checks in variables API --- lib/api/variables.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index c70c7cd9d7b..dac2ba679c7 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -36,6 +36,8 @@ module API variables.where(key: variable_id) end + return not_found!('Variable') if variables.empty? + present variables.first, with: Entities::Variable end @@ -51,6 +53,8 @@ module API put ':id/variables/:variable_id' do variable = user_project.variables.where(id: params[:variable_id].to_i).first + return not_found!('Variable') unless variable + variable.key = params[:key] variable.value = params[:value] variable.save! @@ -67,6 +71,9 @@ module API # DELETE /projects/:id/variables/:variable_id delete ':id/variables/:variable_id' do variable = user_project.variables.where(id: params[:variable_id].to_i).first + + return not_found!('Variable') unless variable + variable.destroy end end -- cgit v1.2.1 From a6c60127e3e2966b2f29fa6e6e79503b130c2b02 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:14:02 +0100 Subject: Removed tracking of raw SQL queries This particular setup had 3 problems: 1. Storing SQL queries as tags is very inefficient as InfluxDB ends up indexing every query (and they can get pretty large). Storing these as values instead means we can't always display the SQL as easily. 2. We already instrument ActiveRecord query methods, thus we already have timing information about database queries. 3. SQL obfuscation is difficult to get right and I'd rather not expose sensitive data by accident. --- lib/gitlab/metrics/obfuscated_sql.rb | 47 ------------------------ lib/gitlab/metrics/subscribers/active_record.rb | 48 ------------------------- 2 files changed, 95 deletions(-) delete mode 100644 lib/gitlab/metrics/obfuscated_sql.rb delete mode 100644 lib/gitlab/metrics/subscribers/active_record.rb (limited to 'lib') diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb deleted file mode 100644 index fe97d7a0534..00000000000 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Metrics - # Class for producing SQL queries with sensitive data stripped out. - class ObfuscatedSQL - REPLACEMENT = / - \d+(\.\d+)? # integers, floats - | '.+?' # single quoted strings - | \/.+?(? Date: Thu, 31 Dec 2015 17:47:07 +0100 Subject: Cache InfluxDB settings after the first use This ensures we don't need to load anything from either PostgreSQL or the Rails cache whenever creating new InfluxDB connections. --- lib/gitlab/metrics.rb | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 2d266ccfe9e..c5b98a0115e 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -6,16 +6,21 @@ module Gitlab METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - def self.pool_size - current_application_settings[:metrics_pool_size] || 16 - end - - def self.timeout - current_application_settings[:metrics_timeout] || 10 + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + username: current_application_settings[:metrics_username], + password: current_application_settings[:metrics_password], + port: current_application_settings[:metrics_port] + } end def self.enabled? - current_application_settings[:metrics_enabled] || false + settings[:enabled] || false end def self.mri? @@ -26,8 +31,7 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= - (current_application_settings[:metrics_method_call_threshold] || 10) + @method_call_threshold ||= settings[:method_call_threshold] end def self.pool @@ -90,11 +94,11 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? - @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = current_application_settings[:metrics_host] - user = current_application_settings[:metrics_username] - pw = current_application_settings[:metrics_password] - port = current_application_settings[:metrics_port] + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + user = settings[:username] + pw = settings[:password] + port = settings[:port] InfluxDB::Client. new(udp: { host: host, port: port }, username: user, password: pw) -- cgit v1.2.1 From bd9f86bb8abb4759a0c72f94fb0492b1ff8619b5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:51:12 +0100 Subject: Use separate series for Rails/Sidekiq transactions This removes the need for tagging all metrics with a "process_type" tag. --- lib/gitlab/metrics/metric.rb | 3 +-- lib/gitlab/metrics/rack_middleware.rb | 4 +++- lib/gitlab/metrics/sidekiq_middleware.rb | 4 +++- lib/gitlab/metrics/transaction.rb | 7 +++---- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 753008df99a..8319e628a40 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -19,8 +19,7 @@ module Gitlab { series: @series, tags: @tags.merge( - hostname: Metrics.hostname, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' + hostname: Metrics.hostname ), values: @values, timestamp: @created_at.to_i * 1_000_000_000 diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..bb9e4fcb918 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,6 +4,8 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' + SERIES = 'rails_transactions' + def initialize(app) @app = app end @@ -30,7 +32,7 @@ module Gitlab end def transaction_from_env(env) - trans = Transaction.new + trans = Transaction.new(SERIES) trans.add_tag(:request_method, env['REQUEST_METHOD']) trans.add_tag(:request_uri, env['REQUEST_URI']) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..6e804dd2562 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -4,8 +4,10 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware + SERIES = 'sidekiq_transactions' + def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new(SERIES) begin trans.run { yield } diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index a61dbd989e7..43a7dab5323 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,8 +4,6 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - SERIES = 'transactions' - attr_reader :uuid, :tags def self.current @@ -13,7 +11,8 @@ module Gitlab end # name - The name of this transaction as a String. - def initialize + def initialize(series) + @series = series @metrics = [] @uuid = SecureRandom.uuid @@ -55,7 +54,7 @@ module Gitlab end def track_self - add_metric(SERIES, { duration: duration }, @tags) + add_metric(@series, { duration: duration }, @tags) end def submit -- cgit v1.2.1 From cafc784ee1d5d0a0279077272af8ee435bb110e4 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 31 Dec 2015 17:55:10 +0100 Subject: Removed tracking of hostnames for metrics This isn't hugely useful and mostly wastes InfluxDB space. We can re-add this whenever needed (but only once we really need it). --- lib/gitlab/metrics.rb | 6 ------ lib/gitlab/metrics/metric.rb | 6 ++---- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index c5b98a0115e..ee88ab34d6c 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -38,10 +38,6 @@ module Gitlab @pool end - def self.hostname - @hostname - end - # Returns a relative path and line number based on the last application call # frame. def self.last_relative_application_frame @@ -89,8 +85,6 @@ module Gitlab value.to_s.gsub('=', '\\=') end - @hostname = Socket.gethostname - # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 8319e628a40..7ea9555cc8c 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -17,10 +17,8 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash { - series: @series, - tags: @tags.merge( - hostname: Metrics.hostname - ), + series: @series, + tags: @tags, values: @values, timestamp: @created_at.to_i * 1_000_000_000 } -- cgit v1.2.1 From 937567b767e6d7b34dcaa1d9c83fc75464638683 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 22:30:07 +0100 Subject: Add create feature to variables API --- lib/api/variables.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index dac2ba679c7..fc63ac2f56a 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -41,6 +41,24 @@ module API present variables.first, with: Entities::Variable end + # Create a new variable in project + # + # Parameters: + # id (required) - The ID of a project + # key (required) - The key of variable being created + # value (required) - The value of variable being created + # Example Request: + # POST /projects/:id/variables + post ':id/variables' do + required_attributes! [:key, :value] + + variable = user_project.variables.create(key: params[:key], value: params[:value]) + return render_validation_error!(variable) unless variable.valid? + variable.save! + + present variable, with: Entities::Variable + end + # Update existing variable of a project # # Parameters: @@ -75,6 +93,8 @@ module API return not_found!('Variable') unless variable variable.destroy + + present variable, with: Entities::Variable end end end -- cgit v1.2.1 From 16bd4df083135e2e4a263b2e1bdd71b78a875ef7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 22:59:06 +0100 Subject: Fix a typo in method description --- lib/api/variables.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index fc63ac2f56a..b8bbcb6ce3b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -18,7 +18,7 @@ module API present paginate(variables), with: Entities::Variable end - # Get specifica bariable of a project + # Get specifica variable of a project # # Parameters: # id (required) - The ID of a project -- cgit v1.2.1 From 628297fe5692fc241c93ff34cece71132bfb9aed Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 23:01:28 +0100 Subject: Remove incorrect 'default' values from method description --- lib/api/builds.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index a519224d2b8..92bf849824c 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -9,8 +9,8 @@ module API # Parameters: # id (required) - The ID of a project # scope (optional) - The scope of builds to show (one of: all, finished, running) - # page (optional) - The page number for pagination (default: 1) - # per_page (ooptional) - The value of items per page to show (default 30) + # page (optional) - The page number for pagination + # per_page (ooptional) - The value of items per page to show # Example Request: # GET /projects/:id/builds get ':id/builds' do -- cgit v1.2.1 From 1a1113f7c4ad6e5bceb898593cba6caecbdf0097 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 1 Jan 2016 22:11:34 -0500 Subject: Simplify `ContributionsCalendar#starting_year` and `#starting_month` --- lib/gitlab/contributions_calendar.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end -- cgit v1.2.1 From 086cfc8685a6489ca032899307c77f828f515fbb Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 26 Aug 2015 23:36:17 -0700 Subject: Fix API project lookups when querying with a namespace with dots Attempting to use the /projects/:id API by specifying :id in "namespace/project" format would always result in a 404 if the namespace contained a dot. The reason? From http://guides.rubyonrails.org/routing.html#specifying-constraints: "By default the :id parameter doesn't accept dots - this is because the dot is used as a separator for formatted routes. If you need to use a dot within an :id add a constraint which overrides this - for example id: /[^\/]+/ allows anything except a slash." Closes https://github.com/gitlabhq/gitlabhq/issues/9573 --- lib/api/projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a9e0960872a..0781236cf6d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,7 +3,7 @@ module API class Projects < Grape::API before { authenticate! } - resource :projects do + resource :projects, requirements: { id: /[^\/]+/ } do helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) -- cgit v1.2.1 From 96075be6f4688a59335130dc796132ad4f232442 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 11:37:46 +0100 Subject: Ability to increment custom transaction values This will be used to store/increment the total query/view rendering timings on a per transaction basis. This in turn can greatly reduce the amount of metrics stored. --- lib/gitlab/metrics/transaction.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 43a7dab5323..0aaebf262d4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -19,7 +19,8 @@ module Gitlab @started_at = nil @finished_at = nil - @tags = {} + @values = Hash.new(0) + @tags = {} end def duration @@ -44,6 +45,10 @@ module Gitlab @metrics << Metric.new(series, values, tags) end + def increment(name, value) + @values[name] += value + end + def add_tag(key, value) @tags[key] = value end @@ -54,7 +59,13 @@ module Gitlab end def track_self - add_metric(@series, { duration: duration }, @tags) + values = { duration: duration } + + @values.each do |name, value| + values[name] = value + end + + add_metric(@series, values, @tags) end def submit -- cgit v1.2.1 From 66a997a91403eef62ffd9fb789e899619d021a26 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:14:36 +0100 Subject: Track total query/view timings in transactions --- lib/gitlab/metrics/subscribers/action_view.rb | 1 + lib/gitlab/metrics/subscribers/active_record.rb | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lib/gitlab/metrics/subscribers/active_record.rb (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7e0dcf99d92..7c0105d543a 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,6 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb new file mode 100644 index 00000000000..8008b3bc895 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -0,0 +1,22 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total query duration of a transaction. + class ActiveRecord < ActiveSupport::Subscriber + attach_to :active_record + + def sql(event) + return unless current_transaction + + current_transaction.increment(:sql_duration, event.duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end -- cgit v1.2.1 From 825b46f8a3eb620f99192217d414b72dffe597d7 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:19:45 +0100 Subject: Track total method call times per transaction This makes it easier to see where time is spent without having to aggregate all the individual points in the method_calls series. --- lib/gitlab/metrics/instrumentation.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 06fc2f25948..d9fce2e6758 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -123,6 +123,8 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) -- cgit v1.2.1 From 2ea464bb272fa52ff34a188a921f0bc90811ca45 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 12:45:31 +0100 Subject: Use separate series for Rails/Sidekiq sample stats This removes the need for any tags to differentiate between Sidekiq and Rails statistics while still being able to separate the two. --- lib/gitlab/metrics/sampler.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 998578e1c0a..1ea425bc904 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -50,12 +50,11 @@ module Gitlab end def sample_memory_usage - @metrics << Metric.new('memory_usage', value: System.memory_usage) + add_metric('memory_usage', value: System.memory_usage) end def sample_file_descriptors - @metrics << Metric. - new('file_descriptors', value: System.file_descriptor_count) + add_metric('file_descriptors', value: System.file_descriptor_count) end if Metrics.mri? @@ -69,7 +68,7 @@ module Gitlab counts['Symbol'] = Symbol.all_symbols.length counts.each do |name, count| - @metrics << Metric.new('object_counts', { count: count }, type: name) + add_metric('object_counts', { count: count }, type: name) end end else @@ -91,7 +90,17 @@ module Gitlab stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - @metrics << Metric.new('gc_statistics', stats) + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? end end end -- cgit v1.2.1 From 2ee8f555996baca6b470d223ffad65419b730398 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Jan 2016 13:17:02 +0100 Subject: Automatically prefix transaction series names This ensures Rails and Sidekiq transactions are split into the series "rails_transactions" and "sidekiq_transactions" respectively. --- lib/gitlab/metrics/rack_middleware.rb | 4 +--- lib/gitlab/metrics/sidekiq_middleware.rb | 4 +--- lib/gitlab/metrics/transaction.rb | 15 +++++++++------ 3 files changed, 11 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index bb9e4fcb918..5c0587c4c51 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,8 +4,6 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' - SERIES = 'rails_transactions' - def initialize(app) @app = app end @@ -32,7 +30,7 @@ module Gitlab end def transaction_from_env(env) - trans = Transaction.new(SERIES) + trans = Transaction.new trans.add_tag(:request_method, env['REQUEST_METHOD']) trans.add_tag(:request_uri, env['REQUEST_URI']) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 6e804dd2562..ad441decfa2 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -4,10 +4,8 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware - SERIES = 'sidekiq_transactions' - def call(worker, message, queue) - trans = Transaction.new(SERIES) + trans = Transaction.new begin trans.run { yield } diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 0aaebf262d4..68b86de0655 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -10,9 +10,7 @@ module Gitlab Thread.current[THREAD_KEY] end - # name - The name of this transaction as a String. - def initialize(series) - @series = series + def initialize @metrics = [] @uuid = SecureRandom.uuid @@ -40,9 +38,10 @@ module Gitlab end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) + tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' - @metrics << Metric.new(series, values, tags) + @metrics << Metric.new("#{prefix}#{series}", values, tags) end def increment(name, value) @@ -65,12 +64,16 @@ module Gitlab values[name] = value end - add_metric(@series, values, @tags) + add_metric('transactions', values, @tags) end def submit Metrics.submit_metrics(@metrics.map(&:to_hash)) end + + def sidekiq? + Sidekiq.server? + end end end end -- cgit v1.2.1 From 4b027bc93a7875c3937f6b90ac1049b4a4d72da5 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 4 Jan 2016 14:29:06 +0100 Subject: Add DEBUG_BANZAI_CACHE env var to debug Banzai cache issue. --- lib/banzai/renderer.rb | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..910e1c6994e 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,13 +18,22 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + cacheless = cacheless_render(text, context) + + if cache_key && ENV["DEBUG_BANZAI_CACHE"] + cached = Rails.cache.fetch(cache_key) { cacheless } + + if cached != cacheless + Rails.logger.warn "Banzai cache mismatch" + Rails.logger.warn "Text: #{text.inspect}" + Rails.logger.warn "Context: #{context.inspect}" + Rails.logger.warn "Cache key: #{cache_key.inspect}" + Rails.logger.warn "Cacheless: #{cacheless.inspect}" + Rails.logger.warn "With cache: #{cached.inspect}" end - else - cacheless_render(text, context) end + + cacheless end def self.render_result(text, context = {}) -- cgit v1.2.1 From d9da81f736b770bb44c4869aef5d5c455e74ab7a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 4 Jan 2016 16:38:32 +0100 Subject: Add triggers feature to API --- lib/api/entities.rb | 10 ++++++++++ lib/api/triggers.rb | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..bc0cd76a2b8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -365,5 +365,15 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Trigger < Grape::Entity + expose :id, :token, :created_at, :updated_at, :deleted_at + expose :last_used do |repo_obj, _options| + if repo_obj.respond_to?(:last_trigger_request) + request = repo_obj.last_trigger_request + request.created_at if request + end + end + end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 2781f1cf191..9a1e3fdc976 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -43,6 +43,24 @@ module API render_api_error!(errors, 400) end end + + # Get triggers list + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/triggers + get ':id/triggers' do + authenticate! + authorize_admin_project + + triggers = user_project.triggers.includes(:trigger_requests) + triggers = paginate(triggers) + + present triggers, with: Entities::Trigger + end end end end -- cgit v1.2.1 From f00607431cd13a952731e36701ebc3b39e64d09b Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 11:27:38 +0100 Subject: Add delete feature to triggers API --- lib/api/triggers.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 9a1e3fdc976..3cb7810241f 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -61,6 +61,25 @@ module API present triggers, with: Entities::Trigger end + + # Delete trigger + # + # Parameters: + # id (required) - The ID of a project + # trigger_id - The ID of trigger to delete + # Example Request: + # DELETE /projects/:id/triggers/:trigger_id + delete ':id/triggers/:trigger_id' do + authenticate! + authorize_admin_project + + trigger = user_project.triggers.where(id: params[:trigger_id].to_i).first + return not_found!('Trigger') unless trigger + + trigger.destroy + + present trigger, with: Entities::Trigger + end end end end -- cgit v1.2.1 From 49c8bf4e9b510be51859dcc301cb46b29b750cb0 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 11:44:10 +0100 Subject: Add create feature to triggers API --- lib/api/triggers.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 3cb7810241f..38cf1e9a2e0 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,6 +62,22 @@ module API present triggers, with: Entities::Trigger end + # Create trigger + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/triggers + post ':id/triggers' do + authenticate! + authorize_admin_project + + trigger = user_project.triggers.new + trigger.save + + present trigger, with: Entities::Trigger + end + # Delete trigger # # Parameters: -- cgit v1.2.1 From 8675664655c4e0f1e043afa88ff1fd75ae5a6a9e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 12:25:16 +0100 Subject: Get show details feature to triggers API --- lib/api/triggers.rb | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 38cf1e9a2e0..0e548b936cd 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,6 +62,32 @@ module API present triggers, with: Entities::Trigger end + # Get specific trigger of a project + # + # Parameters: + # id (required) - The ID of a project + # trigger_id (required) - The ID or `token` of a trigger to show; if trigger_id contains only digits it's + # treated as ID other ways it's reated as `key` + # Example Request: + # GET /projects/:id/triggers/:trigger_id + get ':id/triggers/:trigger_id' do + authenticate! + authorize_admin_project + + trigger_id = params[:trigger_id] + triggers = user_project.triggers + triggers = + if trigger_id.match(/^\d+$/) + triggers.where(id: trigger_id.to_i) + else + triggers.where(token: trigger_id) + end + + return not_found!('Trigger') if triggers.empty? + + present triggers.first, with: Entities::Trigger + end + # Create trigger # # Parameters: -- cgit v1.2.1 From f60bceb988bd629f9adecc070ef5579d264f27c6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 13:20:06 +0100 Subject: Add CI data to projcet entity --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..e8154e0f383 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,9 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + + expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex + expose :runners_token end class ProjectMember < UserBasic -- cgit v1.2.1 From e57b506222f535774059cbb0f986621384c5a8f7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 5 Jan 2016 05:30:01 -0800 Subject: Suggest prefacing find command with sudo when base permissions are wrong Closes #5872 --- lib/tasks/gitlab/check.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 0469c5a61c3..2dc2953e328 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,7 +431,7 @@ namespace :gitlab do try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", - "find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" ) for_more_information( see_installation_guide_section "GitLab Shell" -- cgit v1.2.1 From 0a21731e3bc400ceb9898c9efbc2a186f5348e09 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 15:00:25 +0100 Subject: Add ci fields in project create/update feature API --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..563c12e4f74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) + if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..7dd6b133f9b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,10 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -159,7 +162,10 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -215,7 +221,10 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? -- cgit v1.2.1 From 097faeb481db2a4956b41049c041d55f5da4e2c1 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 5 Jan 2016 16:24:42 +0100 Subject: Get "Merge when build succeeds" to work when commits were pushed to MR target branch while builds were running --- lib/api/merge_requests.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3c1c6bda260..5c97fe1c88c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -211,7 +211,7 @@ module API unauthorized! unless merge_request.can_be_merged_by?(current_user) not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged if merge_request.unchecked? + merge_request.check_if_can_be_merged render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? -- cgit v1.2.1 From a862ade55bf68f56734538b40e02e56036f8a1bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 16:36:14 +0100 Subject: Update ./doc/api/ --- lib/api/builds.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 92bf849824c..6b0edcff820 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -8,7 +8,7 @@ module API # # Parameters: # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one of: all, finished, running) + # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) # page (optional) - The page number for pagination # per_page (ooptional) - The value of items per page to show # Example Request: @@ -24,6 +24,7 @@ module API # Parameters: # id (required) - The ID of a project # sha (required) - The SHA id of a commit + # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do -- cgit v1.2.1 From 0b661516324862506d5ec30c44cac704346a90ad Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 5 Jan 2016 16:45:53 +0100 Subject: Remove icon from milestone reference. --- lib/banzai/filter/milestone_reference_filter.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index f99202af5e8..e88b27c1fae 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -3,8 +3,6 @@ require 'banzai' module Banzai module Filter # HTML filter that replaces milestone references with links. - # - # This filter supports cross-project references. class MilestoneReferenceFilter < AbstractReferenceFilter def self.object_class Milestone -- cgit v1.2.1 From 95f1fe724a47b12b1d66a722be13d15c5fbd2688 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 16:52:45 -0200 Subject: Import GitHub Pull Requests into GitLab --- lib/gitlab/github_import/importer.rb | 64 ++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b5720f6e2cb..656fac9ac7e 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -12,7 +12,16 @@ module Gitlab end def execute - #Issues && Comments + import_issues + import_pull_requests + + true + end + + private + + def import_issues + # Issues && Comments client.list_issues(project.import_source, state: :all, sort: :created, direction: :asc).each do |issue| @@ -33,18 +42,59 @@ module Gitlab description: body, title: issue.title, state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) + author_id: gl_author_id(project, issue.user.id) ) end end end - private + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |pull_request| + body = @formatter.author_line(pull_request.user.login) + body += pull_request.body || "" + + source_branch = pull_request.head.ref + target_branch = pull_request.base.ref + + merge_request = MergeRequest.create!( + title: pull_request.title, + description: body, + source_project: project, + source_branch: source_branch, + target_project: project, + target_branch: target_branch, + state: merge_request_state(pull_request), + author_id: gl_author_id(project, pull_request.user.id), + assignee_id: gl_user_id(pull_request.assignee.try(:id)), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + end + end + + def merge_request_state(pull_request) + case true + when pull_request.state == 'closed' && pull_request.merged_at.present? + 'merged' + when pull_request.state == 'closed' + 'closed' + else + 'opened' + end + end + + def gl_author_id(project, github_id) + gl_user_id(github_id) || project.creator_id + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def gl_user_id(github_id) + if github_id + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end end end end -- cgit v1.2.1 From 223f7f53453d544a8c46c75ca9c54b8b60620961 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:08:54 -0200 Subject: Import comments on GitHub Pull Request into GitLab --- lib/gitlab/github_import/importer.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 656fac9ac7e..35a2930d4a1 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -71,9 +71,23 @@ module Gitlab created_at: pull_request.created_at, updated_at: pull_request.updated_at ) + + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end + def format_body(author, body) + @formatter.author_line(author) + (body || "") + end + def merge_request_state(pull_request) case true when pull_request.state == 'closed' && pull_request.merged_at.present? -- cgit v1.2.1 From f19bf0eaa73007081f2ab10f6a0fb176d5356e36 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:09:09 -0200 Subject: Import comments on the diff of a GitHub Pull Request into GitLab --- lib/gitlab/github_import/importer.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 35a2930d4a1..4b753d24601 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -81,6 +81,18 @@ module Gitlab updated_at: c.updated_at ) end + + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end @@ -99,6 +111,10 @@ module Gitlab end end + def generate_line_code(file_path, position) + Gitlab::Diff::LineCode.generate(file_path, position, 0) + end + def gl_author_id(project, github_id) gl_user_id(github_id) || project.creator_id end -- cgit v1.2.1 From d72b25811e3f0b722ae1c0906e2fe7dffd312403 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 19:42:56 -0200 Subject: Doesn't import GitHub PR where branches were no longer available --- lib/gitlab/github_import/importer.rb | 82 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 4b753d24601..f8a9e0d55ab 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -52,50 +52,56 @@ module Gitlab client.pull_requests(project.import_source, state: :all, sort: :created, direction: :asc).each do |pull_request| - body = @formatter.author_line(pull_request.user.login) - body += pull_request.body || "" - - source_branch = pull_request.head.ref - target_branch = pull_request.base.ref - - merge_request = MergeRequest.create!( - title: pull_request.title, - description: body, - source_project: project, - source_branch: source_branch, - target_project: project, - target_branch: target_branch, - state: merge_request_state(pull_request), - author_id: gl_author_id(project, pull_request.user.id), - assignee_id: gl_user_id(pull_request.assignee.try(:id)), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at + source_branch = find_branch(pull_request.head.ref) + target_branch = find_branch(pull_request.base.ref) + + if source_branch && target_branch + # Pull Request + merge_request = MergeRequest.create!( + title: pull_request.title, + description: format_body(pull_request.user.login, pull_request.body), + source_project: project, + source_branch: source_branch.name, + target_project: project, + target_branch: target_branch.name, + state: merge_request_state(pull_request), + author_id: gl_author_id(project, pull_request.user.id), + assignee_id: gl_user_id(pull_request.assignee.try(:id)), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at ) - end - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + # Comments on Pull Request + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + + # Comments on Pull Request diff + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end end end end + def find_branch(name) + project.repository.find_branch(name) + end + def format_body(author, body) @formatter.author_line(author) + (body || "") end -- cgit v1.2.1 From 6c846ef83d51a176002027e89245a4ea62b4f2bf Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 21 Dec 2015 21:17:57 -0200 Subject: Extract methods to import comments on a GitHub Pull Request --- lib/gitlab/github_import/importer.rb | 52 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index f8a9e0d55ab..2c64f5cebc7 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -56,7 +56,6 @@ module Gitlab target_branch = find_branch(pull_request.base.ref) if source_branch && target_branch - # Pull Request merge_request = MergeRequest.create!( title: pull_request.title, description: format_body(pull_request.user.login, pull_request.body), @@ -71,33 +70,38 @@ module Gitlab updated_at: pull_request.updated_at ) - # Comments on Pull Request - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) - end - - # Comments on Pull Request diff - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) - end + import_comments_on_pull_request(merge_request, pull_request) + import_comments_on_pull_request_diff(merge_request, pull_request) end end end + def import_comments_on_pull_request(merge_request, pull_request) + client.issue_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + end + + def import_comments_on_pull_request_diff(merge_request, pull_request) + client.pull_request_comments(project.import_source, pull_request.number).each do |c| + merge_request.notes.create!( + project: project, + note: format_body(c.user.login, c.body), + commit_id: c.commit_id, + line_code: generate_line_code(c.path, c.position), + author_id: gl_author_id(project, c.user.id), + created_at: c.created_at, + updated_at: c.updated_at + ) + end + end + def find_branch(name) project.repository.find_branch(name) end -- cgit v1.2.1 From dc72a8b30502dd28bf850c2dfdbf31b687fde5d3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 23 Dec 2015 15:04:46 -0200 Subject: Refactoring GithubImport::Importer --- lib/gitlab/github_import/comment.rb | 59 ++++++++++++++++++ lib/gitlab/github_import/importer.rb | 73 ++++------------------ lib/gitlab/github_import/pull_request.rb | 103 +++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 61 deletions(-) create mode 100644 lib/gitlab/github_import/comment.rb create mode 100644 lib/gitlab/github_import/pull_request.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/comment.rb b/lib/gitlab/github_import/comment.rb new file mode 100644 index 00000000000..55de78f889d --- /dev/null +++ b/lib/gitlab/github_import/comment.rb @@ -0,0 +1,59 @@ +module Gitlab + module GithubImport + class Comment + attr_reader :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + @formatter.author_line(author) + body + end + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2c64f5cebc7..7c495655012 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -51,80 +51,31 @@ module Gitlab def import_pull_requests client.pull_requests(project.import_source, state: :all, sort: :created, - direction: :asc).each do |pull_request| - source_branch = find_branch(pull_request.head.ref) - target_branch = find_branch(pull_request.base.ref) + direction: :asc).each do |raw_data| + pull_request = PullRequest.new(project, raw_data) - if source_branch && target_branch - merge_request = MergeRequest.create!( - title: pull_request.title, - description: format_body(pull_request.user.login, pull_request.body), - source_project: project, - source_branch: source_branch.name, - target_project: project, - target_branch: target_branch.name, - state: merge_request_state(pull_request), - author_id: gl_author_id(project, pull_request.user.id), - assignee_id: gl_user_id(pull_request.assignee.try(:id)), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - import_comments_on_pull_request(merge_request, pull_request) - import_comments_on_pull_request_diff(merge_request, pull_request) + if pull_request.valid? + merge_request = MergeRequest.create!(pull_request.attributes) + import_comments_on_pull_request(merge_request, raw_data) + import_comments_on_pull_request_diff(merge_request, raw_data) end end end def import_comments_on_pull_request(merge_request, pull_request) - client.issue_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + client.issue_comments(project.import_source, pull_request.number).each do |raw_data| + comment = Comment.new(project, raw_data) + merge_request.notes.create!(comment.attributes) end end def import_comments_on_pull_request_diff(merge_request, pull_request) - client.pull_request_comments(project.import_source, pull_request.number).each do |c| - merge_request.notes.create!( - project: project, - note: format_body(c.user.login, c.body), - commit_id: c.commit_id, - line_code: generate_line_code(c.path, c.position), - author_id: gl_author_id(project, c.user.id), - created_at: c.created_at, - updated_at: c.updated_at - ) + client.pull_request_comments(project.import_source, pull_request.number).each do |raw_data| + comment = Comment.new(project, raw_data) + merge_request.notes.create!(comment.attributes) end end - def find_branch(name) - project.repository.find_branch(name) - end - - def format_body(author, body) - @formatter.author_line(author) + (body || "") - end - - def merge_request_state(pull_request) - case true - when pull_request.state == 'closed' && pull_request.merged_at.present? - 'merged' - when pull_request.state == 'closed' - 'closed' - else - 'opened' - end - end - - def generate_line_code(file_path, position) - Gitlab::Diff::LineCode.generate(file_path, position, 0) - end - def gl_author_id(project, github_id) gl_user_id(github_id) || project.creator_id end diff --git a/lib/gitlab/github_import/pull_request.rb b/lib/gitlab/github_import/pull_request.rb new file mode 100644 index 00000000000..61e846472f2 --- /dev/null +++ b/lib/gitlab/github_import/pull_request.rb @@ -0,0 +1,103 @@ +module Gitlab + module GithubImport + class PullRequest + attr_reader :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def source_project + project + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end -- cgit v1.2.1 From 98909dd12cd27b85921962326bcaf651c092dcd5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 23 Dec 2015 20:02:59 -0200 Subject: Generate separate comments when importing GitHub Issues into GitLab --- lib/gitlab/github_import/base_formatter.rb | 21 +++++ lib/gitlab/github_import/comment.rb | 59 ------------ lib/gitlab/github_import/comment_formatter.rb | 45 +++++++++ lib/gitlab/github_import/importer.rb | 59 ++++-------- lib/gitlab/github_import/issue_formatter.rb | 66 +++++++++++++ lib/gitlab/github_import/pull_request.rb | 103 --------------------- lib/gitlab/github_import/pull_request_formatter.rb | 93 +++++++++++++++++++ 7 files changed, 244 insertions(+), 202 deletions(-) create mode 100644 lib/gitlab/github_import/base_formatter.rb delete mode 100644 lib/gitlab/github_import/comment.rb create mode 100644 lib/gitlab/github_import/comment_formatter.rb create mode 100644 lib/gitlab/github_import/issue_formatter.rb delete mode 100644 lib/gitlab/github_import/pull_request.rb create mode 100644 lib/gitlab/github_import/pull_request_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/comment.rb b/lib/gitlab/github_import/comment.rb deleted file mode 100644 index 55de78f889d..00000000000 --- a/lib/gitlab/github_import/comment.rb +++ /dev/null @@ -1,59 +0,0 @@ -module Gitlab - module GithubImport - class Comment - attr_reader :project, :raw_data - - def initialize(project, raw_data) - @project = project - @raw_data = raw_data - @formatter = Gitlab::ImportFormatter.new - end - - def attributes - { - project: project, - note: note, - commit_id: raw_data.commit_id, - line_code: line_code, - author_id: author_id, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at - } - end - - private - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def line_code - if on_diff? - Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) - end - end - - def on_diff? - raw_data.path && raw_data.position - end - - def note - @formatter.author_line(author) + body - end - - def gl_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) - end - end - end -end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 7c495655012..38ca7372202 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -21,29 +21,17 @@ module Gitlab private def import_issues - # Issues && Comments client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - body = @formatter.author_line(issue.user.login) - body += issue.body || "" + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - if issue.comments > 0 - body += @formatter.comments_header - - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end - - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_author_id(project, issue.user.id) - ) end end end @@ -52,39 +40,30 @@ module Gitlab client.pull_requests(project.import_source, state: :all, sort: :created, direction: :asc).each do |raw_data| - pull_request = PullRequest.new(project, raw_data) + pull_request = PullRequestFormatter.new(project, raw_data) if pull_request.valid? merge_request = MergeRequest.create!(pull_request.attributes) - import_comments_on_pull_request(merge_request, raw_data) - import_comments_on_pull_request_diff(merge_request, raw_data) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) end end end - def import_comments_on_pull_request(merge_request, pull_request) - client.issue_comments(project.import_source, pull_request.number).each do |raw_data| - comment = Comment.new(project, raw_data) - merge_request.notes.create!(comment.attributes) - end - end - - def import_comments_on_pull_request_diff(merge_request, pull_request) - client.pull_request_comments(project.import_source, pull_request.number).each do |raw_data| - comment = Comment.new(project, raw_data) - merge_request.notes.create!(comment.attributes) - end + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) end - def gl_author_id(project, github_id) - gl_user_id(github_id) || project.creator_id + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) end - def gl_user_id(github_id) - if github_id - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/pull_request.rb b/lib/gitlab/github_import/pull_request.rb deleted file mode 100644 index 61e846472f2..00000000000 --- a/lib/gitlab/github_import/pull_request.rb +++ /dev/null @@ -1,103 +0,0 @@ -module Gitlab - module GithubImport - class PullRequest - attr_reader :project, :raw_data - - def initialize(project, raw_data) - @project = project - @raw_data = raw_data - @formatter = Gitlab::ImportFormatter.new - end - - def attributes - { - title: raw_data.title, - description: description, - source_project: source_project, - source_branch: source_branch.name, - target_project: target_project, - target_branch: target_branch.name, - state: state, - author_id: author_id, - assignee_id: assignee_id, - created_at: raw_data.created_at, - updated_at: updated_at - } - end - - def valid? - source_branch.present? && target_branch.present? - end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gl_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - @formatter.author_line(author) + body - end - - def source_project - project - end - - def source_branch - source_project.repository.find_branch(raw_data.head.ref) - end - - def target_project - project - end - - def target_branch - target_project.repository.find_branch(raw_data.base.ref) - end - - def state - @state ||= case true - when raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - when raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at - else - raw_data.updated_at - end - end - - def gl_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) - end - end - end -end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..42dc09c2ac5 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,93 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def number + raw_data.number + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end -- cgit v1.2.1 From 837a9065f0ff192d2efd55edcc2658a92c127b21 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 5 Jan 2016 15:15:36 -0200 Subject: Ensure that we're only importing local pull requests --- lib/gitlab/github_import/importer.rb | 2 +- lib/gitlab/github_import/pull_request_formatter.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 38ca7372202..2b0afbc7b39 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -42,7 +42,7 @@ module Gitlab direction: :asc).each do |raw_data| pull_request = PullRequestFormatter.new(project, raw_data) - if pull_request.valid? + if !pull_request.cross_project? && pull_request.valid? merge_request = MergeRequest.create!(pull_request.attributes) import_comments(pull_request.number, merge_request) import_comments_on_diff(pull_request.number, merge_request) diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 42dc09c2ac5..b7c47958cc7 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -17,6 +17,10 @@ module Gitlab } end + def cross_project? + source_repo.fork == true + end + def number raw_data.number end @@ -57,6 +61,10 @@ module Gitlab project end + def source_repo + raw_data.head.repo + end + def source_branch source_project.repository.find_branch(raw_data.head.ref) end -- cgit v1.2.1 From 045e8cc38c608b46924ce1ef4de3e8c2adefb1ba Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 29 Dec 2015 20:23:07 -0500 Subject: Update version check images to use SVG --- lib/version_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/version_check.rb b/lib/version_check.rb index ea23344948c..91ad07feee5 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -13,6 +13,6 @@ class VersionCheck end def host - 'https://version.gitlab.com/check.png' + 'https://version.gitlab.com/check.svg' end end -- cgit v1.2.1 From 1e6fc0c6a440ad707d990282ab7a93c178e35cfa Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 30 Dec 2015 16:44:22 -0500 Subject: Define a limited set of filters for SingleLinePipeline Removes the following filters from its parent GfmPipeline: - SyntaxHighlightFilter - UploadLinkFilter - TableOfContentsFilter - LabelReferenceFilter - TaskListFilter Closes #1697 --- lib/banzai/pipeline/single_line_pipeline.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 6725c9039a9..a3c9d4f43aa 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,7 +3,23 @@ require 'banzai' module Banzai module Pipeline class SingleLinePipeline < GfmPipeline + def self.filters + @filters ||= [ + Filter::SanitizationFilter, + Filter::EmojiFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter, + + Filter::UserReferenceFilter, + Filter::IssueReferenceFilter, + Filter::ExternalIssueReferenceFilter, + Filter::MergeRequestReferenceFilter, + Filter::SnippetReferenceFilter, + Filter::CommitRangeReferenceFilter, + Filter::CommitReferenceFilter, + ] + end end end end -- cgit v1.2.1 From b9ed3961b55cf3dbc1a6d4c841d295dd23161c90 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 6 Jan 2016 13:25:13 +0100 Subject: Revert "Add DEBUG_BANZAI_CACHE env var to debug Banzai cache issue." This reverts commit 4b027bc93a7875c3937f6b90ac1049b4a4d72da5. --- lib/banzai/renderer.rb | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 910e1c6994e..115ae914524 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,5 +1,7 @@ module Banzai module Renderer + CACHE_ENABLED = false + # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -18,22 +20,13 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - cacheless = cacheless_render(text, context) - - if cache_key && ENV["DEBUG_BANZAI_CACHE"] - cached = Rails.cache.fetch(cache_key) { cacheless } - - if cached != cacheless - Rails.logger.warn "Banzai cache mismatch" - Rails.logger.warn "Text: #{text.inspect}" - Rails.logger.warn "Context: #{context.inspect}" - Rails.logger.warn "Cache key: #{cache_key.inspect}" - Rails.logger.warn "Cacheless: #{cacheless.inspect}" - Rails.logger.warn "With cache: #{cached.inspect}" + if cache_key && CACHE_ENABLED + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) end + else + cacheless_render(text, context) end - - cacheless end def self.render_result(text, context = {}) -- cgit v1.2.1 From cf19efec3ac0ab4510359dd71df3d511762230c3 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 6 Jan 2016 13:26:02 +0100 Subject: Revert "Temporarily disable Markdown caching" This reverts commit d337d5e7137d9b3fd0f9b8890a3ba9296323acc7. --- lib/banzai/renderer.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..891c0fd7749 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,7 +18,7 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED + if cache_key Rails.cache.fetch(cache_key) do cacheless_render(text, context) end -- cgit v1.2.1 From 7549102bb727daecc51da84af39956b32fc41537 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 6 Jan 2016 15:30:02 +0100 Subject: Store SQL/view timings in milliseconds Transaction timings are also already stored in milliseconds, this keeps things consistent. --- lib/gitlab/metrics/subscribers/action_view.rb | 8 ++++++-- lib/gitlab/metrics/subscribers/active_record.rb | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..84d9e383625 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,7 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) - current_transaction.increment(:view_duration, event.duration) + current_transaction.increment(:view_duration, duration(event)) current_transaction.add_metric(SERIES, values, tags) end @@ -28,7 +28,7 @@ module Gitlab end def values_for(event) - { duration: event.duration } + { duration: duration(event) } end def tags_for(event) @@ -48,6 +48,10 @@ module Gitlab def current_transaction Transaction.current end + + def duration(event) + event.duration * 1000.0 + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 8008b3bc895..6fa73e7a3be 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return unless current_transaction - current_transaction.increment(:sql_duration, event.duration) + current_transaction.increment(:sql_duration, duration(event)) end private @@ -16,6 +16,10 @@ module Gitlab def current_transaction Transaction.current end + + def duration(event) + event.duration * 1000.0 + end end end end -- cgit v1.2.1 From 8fdc00bd4c59183a20a60a6b93228268230bbd2e Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 6 Jan 2016 17:49:02 +0100 Subject: Remove InfluxDB username/password InfluxDB over UDP doesn't use authentication, thus there's no need for these settings. --- lib/gitlab/metrics.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index ee88ab34d6c..44356a0e42c 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,8 +13,6 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - username: current_application_settings[:metrics_username], - password: current_application_settings[:metrics_password], port: current_application_settings[:metrics_port] } end @@ -90,12 +88,10 @@ module Gitlab if enabled? @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do host = settings[:host] - user = settings[:username] - pw = settings[:password] port = settings[:port] InfluxDB::Client. - new(udp: { host: host, port: port }, username: user, password: pw) + new(udp: { host: host, port: port }) end end end -- cgit v1.2.1 From ba9799b42f5bc861df1fc5d41c149cf72e9daf04 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 10:39:02 +0100 Subject: Update ./doc/api --- lib/api/projects.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dd6b133f9b..31b081266a8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -98,6 +98,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -146,6 +149,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -207,6 +213,9 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do -- cgit v1.2.1 From 7ed3a5a240e4997b24d11b96e27126dfaa575abe Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 11:47:06 +0100 Subject: Revert "Store SQL/view timings in milliseconds" This reverts commit 7549102bb727daecc51da84af39956b32fc41537. Apparently I was wrong about ActiveSupport::Notifications::Event#duration returning the duration in seconds, instead it returns it in milliseconds already. --- lib/gitlab/metrics/subscribers/action_view.rb | 8 ++------ lib/gitlab/metrics/subscribers/active_record.rb | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 84d9e383625..7c0105d543a 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,7 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) - current_transaction.increment(:view_duration, duration(event)) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end @@ -28,7 +28,7 @@ module Gitlab end def values_for(event) - { duration: duration(event) } + { duration: event.duration } end def tags_for(event) @@ -48,10 +48,6 @@ module Gitlab def current_transaction Transaction.current end - - def duration(event) - event.duration * 1000.0 - end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 6fa73e7a3be..8008b3bc895 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return unless current_transaction - current_transaction.increment(:sql_duration, duration(event)) + current_transaction.increment(:sql_duration, event.duration) end private @@ -16,10 +16,6 @@ module Gitlab def current_transaction Transaction.current end - - def duration(event) - event.duration * 1000.0 - end end end end -- cgit v1.2.1 From 364b07cff0183956ea11962b94c70448767351d3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 12:44:15 +0100 Subject: Removed UUIDs from metrics transactions While useful for finding out what methods/views belong to a transaction this might result in too much data being stored in InfluxDB. --- lib/gitlab/metrics/transaction.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 68b86de0655..306656d30fe 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :uuid, :tags + attr_reader :tags def self.current Thread.current[THREAD_KEY] @@ -12,7 +12,6 @@ module Gitlab def initialize @metrics = [] - @uuid = SecureRandom.uuid @started_at = nil @finished_at = nil @@ -38,7 +37,6 @@ module Gitlab end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) prefix = sidekiq? ? 'sidekiq_' : 'rails_' @metrics << Metric.new("#{prefix}#{series}", values, tags) -- cgit v1.2.1 From 7b10cb6f0f5758c17dd950587982ff400d7aa971 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 7 Jan 2016 13:05:00 +0100 Subject: Store request methods/URIs as values Since filtering by these values is very rare (they're mostly just displayed as-is) we don't need to waste any index space by saving them as tags. By storing them as values we also greatly reduce the number of series in InfluxDB. --- lib/gitlab/metrics/rack_middleware.rb | 4 ++-- lib/gitlab/metrics/transaction.rb | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..e7a2f26d48b 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -32,8 +32,8 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.add_tag(:request_method, env['REQUEST_METHOD']) - trans.add_tag(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_method, env['REQUEST_METHOD']) trans end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 306656d30fe..73131cc6ef2 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags + attr_reader :tags, :values def self.current Thread.current[THREAD_KEY] @@ -46,6 +46,10 @@ module Gitlab @values[name] += value end + def set(name, value) + @values[name] = value + end + def add_tag(key, value) @tags[key] = value end -- cgit v1.2.1 From 1886d727f738895bb552151d59d4024f405522e2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 13:37:14 +0100 Subject: Add API project upload endpoint --- lib/api/projects.rb | 12 +++++++++++- lib/gitlab/email/receiver.rb | 7 ++----- lib/gitlab/fogbugz_import/importer.rb | 4 +--- 3 files changed, 14 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..8b1390e3289 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -269,7 +269,7 @@ module API # Remove a forked_from relationship # # Parameters: - # id: (required) - The ID of the project being marked as a fork + # id: (required) - The ID of the project being marked as a fork # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do @@ -278,6 +278,16 @@ module API user_project.forked_project_link.destroy end end + + # Upload a file + # + # Parameters: + # id: (required) - The ID of the project + # file: (required) - The file to be uploaded + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + # search for projects current_user has access to # # Parameters: diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2b252b32887..2ca21af5bc8 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -74,7 +74,7 @@ module Gitlab def sent_notification return nil unless reply_key - + SentNotification.for(reply_key) end @@ -82,10 +82,7 @@ module Gitlab attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) attachments.each do |link| - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - reply << "\n\n#{text}" + reply << "\n\n#{link[:markdown]}" end reply diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 403ebeec474..d5f755f90e5 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,9 +232,7 @@ module Gitlab return nil if res.nil? - text = "[#{res['alt']}](#{res['url']})" - text = "!#{text}" if res['is_image'] - text + text = res['markdown'] end def build_attachment_url(rel_url) -- cgit v1.2.1 From 706d7eb0b7a1d54604a11e1202e44069c6acccee Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 13:37:59 +0100 Subject: Satisfy Rubocp --- lib/gitlab/fogbugz_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index d5f755f90e5..0e6bee732f1 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,7 +232,7 @@ module Gitlab return nil if res.nil? - text = res['markdown'] + res['markdown'] end def build_attachment_url(rel_url) -- cgit v1.2.1 From b60c146267dfa8dc1c170426e1817c6b2a168d1a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 13:49:38 +0100 Subject: Change :variable_id to :key as resource ID in API --- lib/api/variables.rb | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index b8bbcb6ce3b..cc038e5731d 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -22,19 +22,12 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated - # as ID other ways it's treated as `key` + # key (required) - The `key` of variable # Example Request: - # GET /projects/:id/variables/:variable_id - get ':id/variables/:variable_id' do - variable_id = params[:variable_id] - variables = user_project.variables - variables = - if variable_id.match(/^\d+$/) - variables.where(id: variable_id.to_i) - else - variables.where(key: variable_id) - end + # GET /projects/:id/variables/:key + get ':id/variables/:key' do + key = params[:key] + variables = user_project.variables.where(key: key) return not_found!('Variable') if variables.empty? @@ -45,8 +38,8 @@ module API # # Parameters: # id (required) - The ID of a project - # key (required) - The key of variable being created - # value (required) - The value of variable being created + # key (required) - The key of variable + # value (required) - The value of variable # Example Request: # POST /projects/:id/variables post ':id/variables' do @@ -63,17 +56,15 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID of a variable - # key (optional) - new value for `key` field of variable - # value (optional) - new value for `value` field of variable + # key (optional) - The `key` of variable + # value (optional) - New value for `value` field of variable # Example Request: - # PUT /projects/:id/variables/:variable_id - put ':id/variables/:variable_id' do - variable = user_project.variables.where(id: params[:variable_id].to_i).first + # PUT /projects/:id/variables/:key + put ':id/variables/:key' do + variable = user_project.variables.where(key: params[:key]).first return not_found!('Variable') unless variable - variable.key = params[:key] variable.value = params[:value] variable.save! @@ -84,11 +75,11 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID of a variable + # key (required) - The ID of a variable # Exanoke Reqyest: - # DELETE /projects/:id/variables/:variable_id - delete ':id/variables/:variable_id' do - variable = user_project.variables.where(id: params[:variable_id].to_i).first + # DELETE /projects/:id/variables/:key + delete ':id/variables/:key' do + variable = user_project.variables.where(key: params[:key]).first return not_found!('Variable') unless variable -- cgit v1.2.1 From b60445906849e84ff52ac6a5d7d501bb5a21eb60 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 14:10:49 +0100 Subject: Update ./doc/api --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f71d072f269..db3164d9d9c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,7 +367,7 @@ module API end class Variable < Grape::Entity - expose :id, :key, :value + expose :key, :value end end end -- cgit v1.2.1 From c3865bda0257506e3e9ba8352913eb288f0e3e34 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 14:04:01 +0100 Subject: Properly set task-list class on single item task lists --- lib/banzai/filter/task_list_filter.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index bdf7c2ebdfc..d0ce13003a5 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -12,13 +12,18 @@ module Banzai # # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class(node, *new_class_names) + def add_css_class_with_fix(node, *new_class_names) if new_class_names.include?('task-list') - super if node.children.any? { |c| c['class'] == 'task-list-item' } - else - super + # Don't add class to all lists + return + elsif new_class_names.include?('task-list-item') + add_css_class_without_fix(node.parent, 'task-list') end + + add_css_class_without_fix(node, *new_class_names) end + + alias_method_chain :add_css_class, :fix end end end -- cgit v1.2.1 From e0ec69d919cb44194e76034f2324ec0d4f5f1df6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 18:48:33 +0100 Subject: Change 'trigger_id' to 'token' as resource ID in triggers API --- lib/api/entities.rb | 2 +- lib/api/triggers.rb | 25 ++++++++----------------- 2 files changed, 9 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index bc0cd76a2b8..37c483b45ec 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,7 +367,7 @@ module API end class Trigger < Grape::Entity - expose :id, :token, :created_at, :updated_at, :deleted_at + expose :token, :created_at, :updated_at, :deleted_at expose :last_used do |repo_obj, _options| if repo_obj.respond_to?(:last_trigger_request) request = repo_obj.last_trigger_request diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 0e548b936cd..25bb8aef20b 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -66,23 +66,14 @@ module API # # Parameters: # id (required) - The ID of a project - # trigger_id (required) - The ID or `token` of a trigger to show; if trigger_id contains only digits it's - # treated as ID other ways it's reated as `key` + # token (required) - The `token` of a trigger # Example Request: - # GET /projects/:id/triggers/:trigger_id - get ':id/triggers/:trigger_id' do + # GET /projects/:id/triggers/:token + get ':id/triggers/:token' do authenticate! authorize_admin_project - trigger_id = params[:trigger_id] - triggers = user_project.triggers - triggers = - if trigger_id.match(/^\d+$/) - triggers.where(id: trigger_id.to_i) - else - triggers.where(token: trigger_id) - end - + triggers = user_project.triggers.where(token: params[:token]) return not_found!('Trigger') if triggers.empty? present triggers.first, with: Entities::Trigger @@ -108,14 +99,14 @@ module API # # Parameters: # id (required) - The ID of a project - # trigger_id - The ID of trigger to delete + # token (required) - The `token` of a trigger # Example Request: - # DELETE /projects/:id/triggers/:trigger_id - delete ':id/triggers/:trigger_id' do + # DELETE /projects/:id/triggers/:token + delete ':id/triggers/:token' do authenticate! authorize_admin_project - trigger = user_project.triggers.where(id: params[:trigger_id].to_i).first + trigger = user_project.triggers.where(token: params[:token]).first return not_found!('Trigger') unless trigger trigger.destroy -- cgit v1.2.1 From 69209612e1793fcebcdb784074056d7a02b0f6f7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 22 Dec 2015 12:15:06 -0800 Subject: Suppress e-mails on failed builds if allow_failure is set Every time I push to GitLab, I get > 2 emails saying a spec failed when I don't care about benchmarks and other specs that have `allow_failure` set to `true`. --- lib/gitlab/build_data_builder.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 86bfa0a4378..34e949130da 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -23,6 +23,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_allow_failure: build.allow_failure, # TODO: do we still need it? project_id: project.id, -- cgit v1.2.1 From 4c90ed52fe6ff7b1155088c46460c411e121feb3 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Fri, 8 Jan 2016 10:10:04 +0100 Subject: Delete tag via API --- lib/api/tags.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'lib') diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 47621f443e6..2d8a9e51bb9 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -40,6 +40,27 @@ module API end end + # Delete tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # DELETE /projects/:id/repository/tags/:tag + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + authorize_push_project + result = DeleteTagService.new(user_project, current_user). + execute(params[:tag_name]) + + if result[:status] == :success + { + tag_name: params[:tag_name] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + # Add release notes to tag # # Parameters: -- cgit v1.2.1 From 549a2fa7873366b52e9ba3caa849073b7b958b73 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 8 Jan 2016 14:01:31 +0100 Subject: Modify builds scope filtering in builds API --- lib/api/builds.rb | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 6b0edcff820..6aae1856953 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -8,9 +8,8 @@ module API # # Parameters: # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) - # page (optional) - The page number for pagination - # per_page (ooptional) - The value of items per page to show + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) # Example Request: # GET /projects/:id/builds get ':id/builds' do @@ -24,7 +23,8 @@ module API # Parameters: # id (required) - The ID of a project # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do @@ -112,14 +112,21 @@ module API end def filter_builds(builds, scope) - case scope - when 'finished' - builds.finished - when 'running' - builds.running - else - builds - end + available_scopes = Ci::Build.available_statuses + scope = + if scope.is_a?(String) || scope.is_a?(Symbol) + available_scopes & [scope.to_s] + elsif scope.is_a?(Array) + available_scopes & scope + elsif scope.respond_to?(:to_h) + available_scopes & scope.to_h.values + else + [] + end + + return builds if scope.empty? + + builds.where(status: scope) end def authorize_manage_builds! -- cgit v1.2.1 From 8d7a968d6dbc70d31953bac5262c86c4ca9dc7d9 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 8 Jan 2016 14:31:39 +0100 Subject: Handle missing DBs in connect_to_db? This ensures CurrentSettings#connect_to_db? returns "false" in the event of a database not existing, instead of raising an error. --- lib/gitlab/current_settings.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7a86c09158e..7f938780ab1 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -41,6 +41,9 @@ module Gitlab use_db && ActiveRecord::Base.connection.active? && !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') + + rescue ActiveRecord::NoDatabaseError + false end end end -- cgit v1.2.1 From 0614793b38db4711053cbcb4fa80d9c8cc492eec Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 8 Jan 2016 17:38:53 +0100 Subject: DRY up upload and download services --- lib/gitlab/fogbugz_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 0e6bee732f1..db580b5e578 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,7 +232,7 @@ module Gitlab return nil if res.nil? - res['markdown'] + res[:markdown] end def build_attachment_url(rel_url) -- cgit v1.2.1 From 6e7db8e23e169bcbf0847ece27b9e44e00ae572b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 30 Dec 2015 16:52:02 -0200 Subject: Prevent ldap_blocked users from being blocked/unblocked by the API --- lib/api/users.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 0d7813428e2..01fd90139b0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -284,10 +284,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user + if !user + not_found!('User') + elsif !user.ldap_blocked? user.block else - not_found!('User') + forbidden!('LDAP blocked users cannot be modified by the API') end end @@ -299,10 +301,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user + if !user + not_found!('User') + elsif !user.ldap_blocked? user.activate else - not_found!('User') + forbidden!('LDAP blocked users cannot be unblocked by the API') end end end -- cgit v1.2.1 From d6dc088affeee4568e771e1d7894e0bcdb955af8 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 30 Dec 2015 20:56:26 -0200 Subject: LDAP synchronization block/unblock new states --- lib/gitlab/ldap/access.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..76cb48d7aa6 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -37,15 +37,15 @@ module Gitlab # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block + user.ldap_block false else - user.activate if user.blocked? && !ldap_config.block_auto_created_users + user.activate if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.block + user.ldap_block false end rescue -- cgit v1.2.1 From 47e4613f4adc2d6ef4b066a87ec772ef8044bdd5 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 7 Jan 2016 14:01:01 -0200 Subject: Code style fixes and some code simplified --- lib/gitlab/ldap/access.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 76cb48d7aa6..ebd9260ad5d 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -40,7 +40,9 @@ module Gitlab user.ldap_block false else - user.activate if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? + if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? + user.activate + end true end else -- cgit v1.2.1 From 1eb7b5ee8d5afeeea74ccbd5627e5a235dffe9fd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 8 Jan 2016 22:57:42 +0100 Subject: Modify entities for builds API --- lib/api/builds.rb | 16 +++++++++++----- lib/api/entities.rb | 37 ++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 6aae1856953..33e6ed24101 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -15,7 +15,9 @@ module API get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get builds for a specific commit of a project @@ -33,7 +35,8 @@ module API builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get a specific build of a project @@ -47,7 +50,8 @@ module API build = get_build(params[:build_id]) return not_found!(build) unless build - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get a trace of a specific build of a project @@ -84,7 +88,8 @@ module API build.cancel - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Retry a specific build of a project @@ -102,7 +107,8 @@ module API build = Ci::Build.retry(build) - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f21da54b8fc..cb00b392db9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -366,16 +366,8 @@ module API expose :id, :variables end - class CiCommit < Grape::Entity + class Runner < Grape::Entity expose :id - expose :ref - expose :sha - expose :committed_at - end - - class CiRunner < Grape::Entity - expose :id - expose :token expose :description expose :active expose :is_shared @@ -383,16 +375,23 @@ module API end class Build < Grape::Entity - expose :id - expose :status - expose :stage - expose :name - expose :ref - expose :commit, with: CiCommit - expose :runner, with: CiRunner - expose :created_at - expose :started_at - expose :finished_at + expose :id, :status, :stage, :name, :ref, :tag, :coverage, :user + expose :created_at, :started_at, :finished_at + expose :download_url do |repo_obj, options| + if options[:user_can_download_artifacts] + repo_obj.download_url + else + nil + end + end + expose :commit, with: RepoCommit do |repo_obj, _options| + if repo_obj.respond_to?(:commit) + repo_obj.commit.commit_data + else + nil + end + end + expose :runner, with: Runner end end end -- cgit v1.2.1 From 4eb27d7c72d57015c7551a00e34a54cefc2d3db9 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 8 Jan 2016 23:33:45 +0100 Subject: Add some modifications to builds API and specs --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 33e6ed24101..05d1b8d92ea 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -103,7 +103,7 @@ module API authorize_manage_builds! build = get_build(params[:build_id]) - return not_found!(build) unless build && build.retryable? + return forbidden!('Build is not retryable') unless build && build.retryable? build = Ci::Build.retry(build) -- cgit v1.2.1 From 96bbc145f31ad029e080ad8903445d81d6c31968 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 10:20:45 +0100 Subject: Change commit builds URL in builds API --- lib/api/builds.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 05d1b8d92ea..8c21754596b 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -28,8 +28,8 @@ module API # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; # if none provided showing all builds) # Example Request: - # GET /projects/:id/builds/commit/:sha - get ':id/builds/commit/:sha' do + # GET /projects/:id/repository/commits/:sha/builds + get ':id/repository/commits/:sha/builds' do commit = user_project.ci_commits.find_by_sha(params[:sha]) return not_found! unless commit -- cgit v1.2.1 From 4e70f2519bba83d5f9d6fd0bed80e9837e8b5fc5 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 11:15:04 +0100 Subject: Update ./doc/api/builds.md --- lib/api/entities.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cb00b392db9..a1a886d6fea 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -375,8 +375,9 @@ module API end class Build < Grape::Entity - expose :id, :status, :stage, :name, :ref, :tag, :coverage, :user + expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at + expose :user, with: UserFull expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url -- cgit v1.2.1 From 67aa0b8c4cbf762211ad178efb537f1649d91776 Mon Sep 17 00:00:00 2001 From: Drew Blessing Date: Thu, 31 Dec 2015 13:22:51 -0600 Subject: Optimize LDAP and add a search timeout --- lib/gitlab/ldap/access.rb | 8 ++++++-- lib/gitlab/ldap/adapter.rb | 24 +++++++++++++++--------- lib/gitlab/ldap/config.rb | 4 ++++ 3 files changed, 25 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..b2bdbc10d7f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,7 +32,7 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap -- cgit v1.2.1 From fc4108b38b5d0a7fc755c1977663f516b5dea0bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:27:20 +0100 Subject: Modify CI features in projects API --- lib/api/entities.rb | 4 +--- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 41 ++++++++++++++--------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8154e0f383..e3bc3316ce5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,9 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } - - expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex - expose :runners_token + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 563c12e4f74..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) + if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 31b081266a8..28c3583f351 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,9 +99,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -117,14 +115,12 @@ module API :namespace_id, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -149,9 +145,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -168,14 +161,12 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -194,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -213,9 +205,6 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do @@ -230,10 +219,7 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :visibility_level] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -247,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end -- cgit v1.2.1 From 5dcfd7d06754fecb245475cc005c112677693801 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:49:25 +0100 Subject: Add TODO notice to build trace feature in CI API --- lib/api/builds.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 8c21754596b..d3f4e33ebbf 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -61,6 +61,10 @@ module API # build_id (required) - The ID of a build # Example Request: # GET /projects/:id/build/:build_id/trace + # + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. get ':id/builds/:build_id/trace' do build = get_build(params[:build_id]) return not_found!(build) unless build -- cgit v1.2.1 From 35b501f30ae9e121151ad6a2140d036e5ef3b0f5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 11 Jan 2016 16:51:01 +0100 Subject: Tag all transaction metrics with an "action" tag Without this it's impossible to find out what methods/views/queries are executed by a certain controller or Sidekiq worker. While this will increase the total number of series it should stay within reasonable limits due to the amount of "actions" being small enough. --- lib/gitlab/metrics/rack_middleware.rb | 6 ++---- lib/gitlab/metrics/sidekiq_middleware.rb | 7 +------ lib/gitlab/metrics/transaction.rb | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e7a2f26d48b..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -39,10 +39,8 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 73131cc6ef2..86606b1c6d6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -6,11 +6,15 @@ module Gitlab attr_reader :tags, :values + attr_accessor :action + def self.current Thread.current[THREAD_KEY] end - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] @started_at = nil @@ -18,6 +22,7 @@ module Gitlab @values = Hash.new(0) @tags = {} + @action = action end def duration @@ -70,7 +75,15 @@ module Gitlab end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) end def sidekiq? -- cgit v1.2.1 From 41bcab118ae8be1baee0d750d99f46f584bd2fd4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:05:43 +0800 Subject: Fix #9963 reference_filter "Encoding::CompatibilityError" bug with some complex URL; --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 7198a8b03e2..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 932a247f5fb4a14b3e096ec88a73f8fce481eebb Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:32:25 +0800 Subject: Use CGI.escape instead of URI.escape, because URI is obsoleted. ref: https://github.com/ruby/ruby/commit/238b979f1789f95262a267d8df6239806f2859cc --- lib/gitlab/gitlab_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) -- cgit v1.2.1 From 5679ee0120ab45829b847d55d3a2253735856b3f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 14:59:30 +0100 Subject: Track memory allocated during a transaction This gives a very rough estimate of how much memory is allocated during a transaction. This only works reliably when using a single-threaded application server and a Ruby implementation with a GIL as otherwise memory allocated by other threads might skew the statistics. Sadly there's no way around this as Ruby doesn't provide a reliable way of gathering accurate object sizes upon allocation on a per-thread basis. --- lib/gitlab/metrics/transaction.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 86606b1c6d6..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -23,20 +23,29 @@ module Gitlab @values = Hash.new(0) @tags = {} @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end @@ -65,7 +74,7 @@ module Gitlab end def track_self - values = { duration: duration } + values = { duration: duration, allocated_memory: allocated_memory } @values.each do |name, value| values[name] = value -- cgit v1.2.1 From ac6a10f3e88c5d2081b8638df63016089517a844 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Tue, 12 Jan 2016 12:29:10 -0200 Subject: Codestyle changes --- lib/api/users.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 01fd90139b0..fd2128bd179 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -303,10 +303,10 @@ module API if !user not_found!('User') - elsif !user.ldap_blocked? - user.activate - else + elsif user.ldap_blocked? forbidden!('LDAP blocked users cannot be unblocked by the API') + else + user.activate end end end -- cgit v1.2.1 From 355c341fe7c6b7c503386cf0b0e39cc02dbc8902 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 15:41:22 +0100 Subject: Stop tracking call stacks for instrumented views Where a vew is called from doesn't matter as much. We already know what action they belong to and this is more than enough information. By removing the file/line number from the list of tags we should also be able to reduce the number of series stored in InfluxDB. --- lib/gitlab/metrics.rb | 14 -------------- lib/gitlab/metrics/subscribers/action_view.rb | 10 +--------- 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 44356a0e42c..cdf7c168ff2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -36,20 +36,6 @@ module Gitlab @pool end - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -33,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction -- cgit v1.2.1 From ab2c6cc01ff26e07db15110e037e72159c48dc53 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 12 Jan 2016 18:32:18 +0100 Subject: Add some fixes --- lib/api/api.rb | 1 - lib/api/entities.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 266b5f48f8f..1a1340c428c 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,7 +54,6 @@ module API mount Keys mount Tags mount Triggers - mount Builds end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index a1a886d6fea..e19f9f8d75a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -377,7 +377,7 @@ module API class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at - expose :user, with: UserFull + expose :user, with: User expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url -- cgit v1.2.1 From efb3395b4fc0425ebbc2437ad03f0cd5fc851863 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 12 Jan 2016 19:32:44 +0100 Subject: Remove blank line --- lib/api/api.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index a9e1913f0f2..098dd975840 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,7 +54,6 @@ module API mount Keys mount Tags mount Triggers - mount Variables end end -- cgit v1.2.1 From a43fd5ce6d369c35b3ea421f045721d20f1c8ada Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 12 Jan 2016 21:34:23 -0500 Subject: Disable colorization if STDOUT is not a tty --- lib/tasks/gitlab/task_helpers.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..8c63877e51c 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,8 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From d44653da1f74c2c15fe7ec3f8aa9b16563ffebd6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 12:16:27 +0100 Subject: Add some fixes after review --- lib/api/entities.rb | 8 +------- lib/api/triggers.rb | 11 +++++------ 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 37c483b45ec..1108277aabf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,13 +367,7 @@ module API end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at - expose :last_used do |repo_obj, _options| - if repo_obj.respond_to?(:last_trigger_request) - request = repo_obj.last_trigger_request - request.created_at if request - end - end + expose :token, :created_at, :updated_at, :deleted_at, :last_used end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 25bb8aef20b..5e4964f446c 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -73,10 +73,10 @@ module API authenticate! authorize_admin_project - triggers = user_project.triggers.where(token: params[:token]) - return not_found!('Trigger') if triggers.empty? + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger - present triggers.first, with: Entities::Trigger + present trigger, with: Entities::Trigger end # Create trigger @@ -89,8 +89,7 @@ module API authenticate! authorize_admin_project - trigger = user_project.triggers.new - trigger.save + trigger = user_project.triggers.create present trigger, with: Entities::Trigger end @@ -106,7 +105,7 @@ module API authenticate! authorize_admin_project - trigger = user_project.triggers.where(token: params[:token]).first + trigger = user_project.triggers.find_by(token: params[:token].to_s) return not_found!('Trigger') unless trigger trigger.destroy -- cgit v1.2.1 From 23671600150cb022a09a77b8ea56a9465f19a013 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:29:48 +0100 Subject: Make the metrics sampler interval configurable --- lib/gitlab/metrics.rb | 3 ++- lib/gitlab/metrics/sampler.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cdf7c168ff2..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,7 +13,8 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port] + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 } end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 1ea425bc904..c2913841de3 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,7 +7,7 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) + def initialize(interval = Metrics.settings[:sample_interval]) @interval = interval @metrics = [] -- cgit v1.2.1 From df548285804fdc40ac7c4f36601e87a534792a4a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 12:47:11 +0100 Subject: Add some fixes after review --- lib/api/variables.rb | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index cc038e5731d..0c3fb5c8a77 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -27,11 +27,11 @@ module API # GET /projects/:id/variables/:key get ':id/variables/:key' do key = params[:key] - variables = user_project.variables.where(key: key) + variable = user_project.variables.find_by(key: key.to_s) - return not_found!('Variable') if variables.empty? + return not_found!('Variable') unless variable - present variables.first, with: Entities::Variable + present variable, with: Entities::Variable end # Create a new variable in project @@ -46,10 +46,12 @@ module API required_attributes! [:key, :value] variable = user_project.variables.create(key: params[:key], value: params[:value]) - return render_validation_error!(variable) unless variable.valid? - variable.save! - present variable, with: Entities::Variable + if variable.valid? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end end # Update existing variable of a project @@ -61,14 +63,16 @@ module API # Example Request: # PUT /projects/:id/variables/:key put ':id/variables/:key' do - variable = user_project.variables.where(key: params[:key]).first + variable = user_project.variables.find_by(key: params[:key].to_s) return not_found!('Variable') unless variable - variable.value = params[:value] - variable.save! - - present variable, with: Entities::Variable + attrs = attributes_for_keys [:value] + if variable.update(attrs) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end end # Delete existing variable of a project @@ -79,10 +83,9 @@ module API # Exanoke Reqyest: # DELETE /projects/:id/variables/:key delete ':id/variables/:key' do - variable = user_project.variables.where(key: params[:key]).first + variable = user_project.variables.find_by(key: params[:key].to_s) return not_found!('Variable') unless variable - variable.destroy present variable, with: Entities::Variable -- cgit v1.2.1 From 057eb824b5d7f38547506303dc80da6164715420 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:57:46 +0100 Subject: Randomize metrics sample intervals Sampling data at a fixed interval means we can potentially miss data from events occurring between sampling intervals. For example, say we sample data every 15 seconds but Unicorn workers get killed after 10 seconds. In this particular case it's possible to miss interesting data as the sampler will never get to actually submitting data. To work around this (at least for the most part) the sampling interval is randomized as following: 1. Take the user specified sampling interval (15 seconds by default) 2. Divide it by 2 (referred to as "half" below) 3. Generate a range (using a step of 0.1) from -"half" to "half" 4. Every time the sampler goes to sleep we'll grab the user provided interval and add a randomly chosen "adjustment" to it while making sure we don't pick the same value twice in a row. For a specified timeout of 15 this means the actual intervals can be anywhere between 7.5 and 22.5, but never can the same interval be used twice in a row. The rationale behind this change is that on dev.gitlab.org I'm sometimes seeing certain Gitlab::Git/Rugged objects being retained, but only for a few minutes every 24 hours. Knowing the code of Gitlab and how much memory it uses/leaks I suspect we're missing data due to workers getting terminated before the sampler can write its data to InfluxDB. --- lib/gitlab/metrics/sampler.rb | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index c2913841de3..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -8,8 +8,13 @@ module Gitlab class Sampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - @interval = interval - @metrics = [] + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -102,6 +107,23 @@ module Gitlab def sidekiq? Sidekiq.server? end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end end end end -- cgit v1.2.1 From 97338496188add9ec8d192c7e78f6a6040befffa Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 15:17:59 +0100 Subject: Add some fixes after review --- lib/api/builds.rb | 24 +++++++++++++----------- lib/api/entities.rb | 6 ++---- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index d3f4e33ebbf..1337e1bb452 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -35,6 +35,7 @@ module API builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) + present paginate(builds), with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end @@ -118,25 +119,26 @@ module API helpers do def get_build(id) - user_project.builds.where(id: id).first + user_project.builds.find_by(id: id.to_i) end def filter_builds(builds, scope) - available_scopes = Ci::Build.available_statuses + return builds if scope.nil? || scope.empty? + + available_statuses = Ci::Build.available_statuses scope = - if scope.is_a?(String) || scope.is_a?(Symbol) - available_scopes & [scope.to_s] - elsif scope.is_a?(Array) - available_scopes & scope - elsif scope.respond_to?(:to_h) - available_scopes & scope.to_h.values + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values else - [] + ['unknown'] end - return builds if scope.empty? + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? - builds.where(status: scope) + builds.where(status: available_statuses && scope) end def authorize_manage_builds! diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e19f9f8d75a..f0816a46529 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -378,18 +378,16 @@ module API expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :user, with: User + # TODO: download_url in Ci:Build model is an GitLab Web Interface URL, not API URL. We should think on some API + # for downloading of artifacts (see: https://gitlab.com/gitlab-org/gitlab-ce/issues/4255) expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url - else - nil end end expose :commit, with: RepoCommit do |repo_obj, _options| if repo_obj.respond_to?(:commit) repo_obj.commit.commit_data - else - nil end end expose :runner, with: Runner -- cgit v1.2.1 From 9e701ccd48ed442124509aeb68fe6788579efdde Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 18:47:39 +0100 Subject: Fix some typos --- lib/api/variables.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 0c3fb5c8a77..d9a055f6c92 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -18,7 +18,7 @@ module API present paginate(variables), with: Entities::Variable end - # Get specifica variable of a project + # Get specific variable of a project # # Parameters: # id (required) - The ID of a project @@ -80,7 +80,7 @@ module API # Parameters: # id (required) - The ID of a project # key (required) - The ID of a variable - # Exanoke Reqyest: + # Example Request: # DELETE /projects/:id/variables/:key delete ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key].to_s) -- cgit v1.2.1 From 0c10aee59677e2dadfef6538a74fe1e28fcdd37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 13 Jan 2016 19:42:36 +0100 Subject: Ensure the API doesn't return notes that the current user shouldn't see --- lib/api/notes.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 3efdfe2d46e..174473f5371 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -20,7 +20,19 @@ module API # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - present paginate(@noteable.notes), with: Entities::Note + + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note end # Get a single +noteable+ note @@ -35,7 +47,12 @@ module API get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) @note = @noteable.notes.find(params[:note_id]) - present @note, with: Entities::Note + + if @note.cross_reference_not_visible_for?(current_user) + not_found!("Note") + else + present @note, with: Entities::Note + end end # Create a new +noteable+ note -- cgit v1.2.1 From dd6fc01ff8a073880b67a323a547edeb5d63f167 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 14 Jan 2016 03:31:27 -0200 Subject: fixed LDAP activation on login to use new ldap_blocked state --- lib/gitlab/ldap/access.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index ebd9260ad5d..a659d179b5f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -40,9 +40,7 @@ module Gitlab user.ldap_block false else - if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? - user.activate - end + user.activate if user.ldap_blocked? true end else -- cgit v1.2.1 From 2a20c0e590be2f673188f1a0d27409a714800c89 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 15:46:12 +0100 Subject: Prevent circular reference error in Banzai --- lib/banzai/cross_project_reference.rb | 2 -- lib/banzai/filter.rb | 1 - lib/banzai/filter/abstract_reference_filter.rb | 2 -- lib/banzai/filter/autolink_filter.rb | 1 - lib/banzai/filter/commit_range_reference_filter.rb | 2 -- lib/banzai/filter/commit_reference_filter.rb | 2 -- lib/banzai/filter/emoji_filter.rb | 1 - lib/banzai/filter/external_issue_reference_filter.rb | 2 -- lib/banzai/filter/external_link_filter.rb | 1 - lib/banzai/filter/issue_reference_filter.rb | 2 -- lib/banzai/filter/label_reference_filter.rb | 2 -- lib/banzai/filter/markdown_filter.rb | 1 - lib/banzai/filter/merge_request_reference_filter.rb | 2 -- lib/banzai/filter/redactor_filter.rb | 1 - lib/banzai/filter/reference_filter.rb | 1 - lib/banzai/filter/reference_gatherer_filter.rb | 1 - lib/banzai/filter/relative_link_filter.rb | 1 - lib/banzai/filter/sanitization_filter.rb | 1 - lib/banzai/filter/snippet_reference_filter.rb | 2 -- lib/banzai/filter/syntax_highlight_filter.rb | 1 - lib/banzai/filter/table_of_contents_filter.rb | 1 - lib/banzai/filter/task_list_filter.rb | 1 - lib/banzai/filter/upload_link_filter.rb | 1 - lib/banzai/filter/user_reference_filter.rb | 2 -- lib/banzai/lazy_reference.rb | 2 -- lib/banzai/pipeline.rb | 2 -- lib/banzai/pipeline/asciidoc_pipeline.rb | 2 -- lib/banzai/pipeline/atom_pipeline.rb | 2 -- lib/banzai/pipeline/base_pipeline.rb | 1 - lib/banzai/pipeline/combined_pipeline.rb | 2 -- lib/banzai/pipeline/description_pipeline.rb | 2 -- lib/banzai/pipeline/email_pipeline.rb | 2 -- lib/banzai/pipeline/full_pipeline.rb | 2 -- lib/banzai/pipeline/gfm_pipeline.rb | 2 -- lib/banzai/pipeline/note_pipeline.rb | 2 -- lib/banzai/pipeline/plain_markdown_pipeline.rb | 2 -- lib/banzai/pipeline/post_process_pipeline.rb | 2 -- lib/banzai/pipeline/reference_extraction_pipeline.rb | 2 -- lib/banzai/pipeline/single_line_pipeline.rb | 2 -- lib/banzai/reference_extractor.rb | 2 -- lib/gitlab/markdown/pipeline.rb | 2 -- lib/gitlab/reference_extractor.rb | 2 -- 42 files changed, 69 deletions(-) (limited to 'lib') diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index ba2866e1efa..0257848b6bc 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Common methods for ReferenceFilters that support an optional cross-project # reference. diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index fd4fe024252..905c4c0144e 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' module Banzai module Filter diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index b2db10e6864..cdbaecf8d90 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # Issues, Merge Requests, Snippets, Commits and Commit Ranges share diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index da4ee80c1b5..856f56fb175 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index e67cd45ab9b..470727ee312 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit range references with links. diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 9e57608b483..713a56ba949 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit references with links. diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 86838e1483c..5952a031626 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,5 +1,4 @@ require 'action_controller' -require 'banzai' require 'gitlab_emoji' require 'html/pipeline/filter' diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 6136e73c096..edc26386903 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces external issue tracker references with links. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index ac87b9820af..8d368f3b9e7 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 51180cb901a..9f08aa36e8b 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces issue references with links. References to diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a3a7a23c1e6..95e7d209119 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces label references with links. diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index d09cf41df39..0659fed1419 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 755b946a34b..57c71708992 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces merge request references with links. References diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 66f77902319..7141ed7c9bd 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 5dd6d2fe3c7..20bd4f7ee6e 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index bef04112919..86d484feb90 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 66f166939e4..41380627d39 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d03e3ae4b3c..3f49d492f2f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 1ad5df96f85..c870a42f741 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces snippet references with links. References to diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index c889cc1e97c..8c5855e5ffc 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 9b3e67206d5..4056dcd6d64 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index d0ce13003a5..66608c9859c 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'task_list/filter' module Banzai diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 1a1d0aad8ca..f642aee0967 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 964ab60f614..24f16f8b547 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces user or group references with links. diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb index 073ec5d9801..1095b4debc7 100644 --- a/lib/banzai/lazy_reference.rb +++ b/lib/banzai/lazy_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai class LazyReference def self.load(refs) diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb index 4e017809d9d..142a9962eb1 100644 --- a/lib/banzai/pipeline.rb +++ b/lib/banzai/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline def self.[](name) diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb index 5e76a817be5..f1331c0ebf9 100644 --- a/lib/banzai/pipeline/asciidoc_pipeline.rb +++ b/lib/banzai/pipeline/asciidoc_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AsciidocPipeline < BasePipeline diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 957f352aec5..9694e4bc23f 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AtomPipeline < FullPipeline diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index cd30009e5c0..db5177db7b3 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline' module Banzai diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb index f3bf1809d18..9485199132e 100644 --- a/lib/banzai/pipeline/combined_pipeline.rb +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline module CombinedPipeline diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 94c2cb165a5..20e24ace352 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class DescriptionPipeline < FullPipeline diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 14356145a35..e47c384afc1 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class EmailPipeline < FullPipeline diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index 72395a5d50e..d47ddfda4be 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 838155e8831..b7a38ea8427 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class GfmPipeline < BasePipeline diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb index 89335143852..7890f20f716 100644 --- a/lib/banzai/pipeline/note_pipeline.rb +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class NotePipeline < FullPipeline diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 998fd75daa2..3fbc681457b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 148f24b6ce1..bd338c045f3 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PostProcessPipeline < BasePipeline diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb index 4f9bc9fcccc..eaddccd30a5 100644 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class ReferenceExtractionPipeline < BasePipeline diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index a3c9d4f43aa..8b84ab401df 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class SingleLinePipeline < GfmPipeline diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 2c197d31898..f4079538ec5 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 8f3f43c0e91..699d8b9fc07 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab module Markdown class Pipeline diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 4164e998dd1..4d830aa45e1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor -- cgit v1.2.1 From 3183092ca94b14d6e61f5e8ba51069554646baf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 14 Jan 2016 12:08:44 +0100 Subject: Add pagination headers to already paginated API resources --- lib/api/helpers.rb | 30 +++++++++++++++++++----------- lib/api/notes.rb | 12 +++--------- 2 files changed, 22 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..312ef90915f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,11 +97,9 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) - add_pagination_headers(paginated, per_page) - - paginated + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end end def authenticate! @@ -327,16 +325,26 @@ module API private - def add_pagination_headers(paginated, per_page) + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) request_url = request.url.split('?').first links = [] - links << %(<#{request_url}?page=#{paginated.current_page - 1}&per_page=#{per_page}>; rel="prev") unless paginated.first_page? - links << %(<#{request_url}?page=#{paginated.current_page + 1}&per_page=#{per_page}>; rel="next") unless paginated.last_page? - links << %(<#{request_url}?page=1&per_page=#{per_page}>; rel="first") - links << %(<#{request_url}?page=#{paginated.total_pages}&per_page=#{per_page}>; rel="last") + links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page? + links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page? + links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first") + links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last") - header 'Link', links.join(', ') + links.join(', ') end def abilities diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..ebd9e97148c 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -22,17 +22,11 @@ module API @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. + # by the current user. notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). + @noteable.notes. reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note + present paginate(Kaminari.paginate_array(notes)), with: Entities::Note end # Get a single +noteable+ note -- cgit v1.2.1 From f5d530865875440d69217cf249715bffaa3d11b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 11:59:10 +0100 Subject: Add implementation of StringPath class `StringPath` class is something similar to Ruby's `Pathname` class, but does not involve any IO operations. `StringPath` objects require passing string representation of path, and array of paths that represents universe to constructor to be intantiated. --- lib/gitlab/string_path.rb | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/gitlab/string_path.rb (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb new file mode 100644 index 00000000000..be65b41dff5 --- /dev/null +++ b/lib/gitlab/string_path.rb @@ -0,0 +1,35 @@ +module Gitlab + ## + # Class that represents a path to a file or directory + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # + class StringPath + def initialize(path, universe) + @path = path + @universe = universe + end + + def absolute? + @path.start_with?('/') + end + + def relative? + !absolute? + end + + def directory? + @path.end_with?('/') + end + + def file? + !directory? + end + + def to_s + @path + end + end +end -- cgit v1.2.1 From 73d2c7a553ca239cdce04af793992fd579ad3e4b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 12:32:21 +0100 Subject: Add new methods to StringPath --- lib/gitlab/string_path.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index be65b41dff5..9ccf54bd62f 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -7,11 +7,17 @@ module Gitlab # # class StringPath + attr_reader :path, :universe + def initialize(path, universe) @path = path @universe = universe end + def to_s + @path + end + def absolute? @path.start_with?('/') end @@ -28,8 +34,17 @@ module Gitlab !directory? end - def to_s - @path + def files + raise NotImplementedError + end + + def basename + name = @path.split(::File::SEPARATOR).last + directory? ? name + ::File::SEPARATOR : name + end + + def ==(other) + @path == other.path && @universe == other.universe end end end -- cgit v1.2.1 From 80a71576ba27d84b3406a8b929328359e2edc9da Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 12:55:50 +0100 Subject: Use `Gitlab::StringPath` in CI build artifacts controller --- lib/gitlab/string_path.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 9ccf54bd62f..3aa6200b572 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -34,6 +34,18 @@ module Gitlab !directory? end + def has_parent? + raise NotImplementedError + end + + def parent + raise NotImplementedError + end + + def directories + raise NotImplementedError + end + def files raise NotImplementedError end -- cgit v1.2.1 From 518b206287318006f9b57382a747b1474b6795a4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 19 Dec 2015 09:31:52 +0100 Subject: Add `parent` iteration implementation to `StringPath` --- lib/gitlab/string_path.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 3aa6200b572..3add56d2213 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -35,11 +35,12 @@ module Gitlab end def has_parent? - raise NotImplementedError + @universe.include?(@path.sub(basename, '')) end def parent - raise NotImplementedError + return nil unless has_parent? + new(@path.sub(basename, '')) end def directories @@ -58,5 +59,11 @@ module Gitlab def ==(other) @path == other.path && @universe == other.universe end + + private + + def new(path) + self.class.new(path, @universe) + end end end -- cgit v1.2.1 From d382335dcd9285c9355ed04dc12c5314bca3c024 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 21 Dec 2015 10:11:15 +0100 Subject: Add implementation of remaining methods in `StringPath` --- lib/gitlab/string_path.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 3add56d2213..a2dc63db2f2 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -43,12 +43,24 @@ module Gitlab new(@path.sub(basename, '')) end + def descendants + return [] unless directory? + children = @universe.select { |entry| entry =~ /^#{@path}.+/ } + children.map { |path| new(path) } + end + + def children + descendants.select { |descendant| descendant.parent == self } + end + def directories - raise NotImplementedError + return [] unless directory? + children.select { |child| child.directory? } end def files - raise NotImplementedError + return [] unless directory? + children.select { |child| child.file? } end def basename -- cgit v1.2.1 From 37b2c5dd5521f25a7195e82538a0ffc528c3ec6d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 21 Dec 2015 12:08:04 +0100 Subject: Add support for root path for `StringPath` --- lib/gitlab/string_path.rb | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index a2dc63db2f2..d165829132a 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -1,6 +1,6 @@ module Gitlab ## - # Class that represents a path to a file or directory + # Class that represents a simplified path to a file or directory # # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. @@ -10,8 +10,9 @@ module Gitlab attr_reader :path, :universe def initialize(path, universe) - @path = path - @universe = universe + @path = prepare(path) + @universe = universe.map { |entry| prepare(entry) } + @universe.unshift('./') unless @universe.include?('./') end def to_s @@ -43,6 +44,15 @@ module Gitlab new(@path.sub(basename, '')) end + def basename + name = @path.split(::File::SEPARATOR).last + directory? ? name + ::File::SEPARATOR : name + end + + def has_descendants? + descendants.any? + end + def descendants return [] unless directory? children = @universe.select { |entry| entry =~ /^#{@path}.+/ } @@ -63,11 +73,6 @@ module Gitlab children.select { |child| child.file? } end - def basename - name = @path.split(::File::SEPARATOR).last - directory? ? name + ::File::SEPARATOR : name - end - def ==(other) @path == other.path && @universe == other.universe end @@ -77,5 +82,10 @@ module Gitlab def new(path) self.class.new(path, @universe) end + + def prepare(path) + return path if path =~ %r{^(/|\.|\.\.)} + path.dup.prepend('./') + end end end -- cgit v1.2.1 From b19e958d86f5363057f006c8dbf9a8e8762618b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 Dec 2015 11:05:22 +0100 Subject: Add support for parent directories in `StringPath` This support is not completed though, as parent directory that is first in collection returned by `directories!` is not iterable yet. --- lib/gitlab/string_path.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index d165829132a..493fceb256d 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,6 +5,7 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # + # TODO: better support for './' and '../' # class StringPath attr_reader :path, :universe @@ -45,10 +46,13 @@ module Gitlab end def basename - name = @path.split(::File::SEPARATOR).last directory? ? name + ::File::SEPARATOR : name end + def name + @path.split(::File::SEPARATOR).last + end + def has_descendants? descendants.any? end @@ -68,6 +72,10 @@ module Gitlab children.select { |child| child.directory? } end + def directories! + has_parent? ? directories.prepend(new(@path + '../')) : directories + end + def files return [] unless directory? children.select { |child| child.file? } -- cgit v1.2.1 From 1cc26e0f0e2ef62fecb688e45bc51c66c8fdf0a7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 Dec 2015 13:02:00 +0100 Subject: Improve performance of `StringPath` --- lib/gitlab/string_path.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 493fceb256d..1eb8162f805 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,15 +5,15 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO: better support for './' and '../' + # TODO: better support for '../' and './' # class StringPath attr_reader :path, :universe def initialize(path, universe) @path = prepare(path) - @universe = universe.map { |entry| prepare(entry) } - @universe.unshift('./') unless @universe.include?('./') + @universe = Set.new(universe.map { |entry| prepare(entry) }) + @universe.add('./') end def to_s @@ -64,7 +64,10 @@ module Gitlab end def children - descendants.select { |descendant| descendant.parent == self } + return [] unless directory? + return @children if @children + children = @universe.select { |entry| entry =~ %r{^#{@path}[^/]+/?$} } + @children = children.map { |path| new(path) } end def directories @@ -85,6 +88,10 @@ module Gitlab @path == other.path && @universe == other.universe end + def inspect + "#{self.class.name}: #{@path}" + end + private def new(path) -- cgit v1.2.1 From c177784d5af6b47ae613f922e075a38fc56ad711 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 29 Dec 2015 11:40:57 +0100 Subject: Use short method call in StringPath instead block --- lib/gitlab/string_path.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 1eb8162f805..af80a502bf6 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -72,7 +72,7 @@ module Gitlab def directories return [] unless directory? - children.select { |child| child.directory? } + children.select(&:directory?) end def directories! @@ -81,7 +81,7 @@ module Gitlab def files return [] unless directory? - children.select { |child| child.file? } + children.select(&:file?) end def ==(other) -- cgit v1.2.1 From 447f56036e837fc9a9c2bcaf382d38dc513a9733 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Dec 2015 09:25:59 +0100 Subject: Use metadata stored in artifacats metadata file --- lib/gitlab/string_path.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index af80a502bf6..bfeb0f852f0 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,7 +5,7 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO: better support for '../' and './' + # TODO, better support for '../' and './' # class StringPath attr_reader :path, :universe -- cgit v1.2.1 From 3de8a4620a70c886c815576dc0a30a745cbb8ccb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Dec 2015 12:21:56 +0100 Subject: Parse artifacts metadata stored in JSON format --- lib/gitlab/string_path.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index bfeb0f852f0..ad68a8ff2d6 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -10,10 +10,11 @@ module Gitlab class StringPath attr_reader :path, :universe - def initialize(path, universe) + def initialize(path, universe, metadata = []) @path = prepare(path) - @universe = Set.new(universe.map { |entry| prepare(entry) }) - @universe.add('./') + @universe = universe.map { |entry| prepare(entry) } + @universe << './' unless @universe.include?('./') + @metadata = metadata end def to_s @@ -84,6 +85,11 @@ module Gitlab children.select(&:file?) end + def metadata + index = @universe.index(@path) + @metadata[index] + end + def ==(other) @path == other.path && @universe == other.universe end -- cgit v1.2.1 From a3191463b60c8ded25a2898d5e5520ae4aff1114 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 2 Jan 2016 09:43:10 +0100 Subject: Add path sanitization to `StringPath` [ci skip] --- lib/gitlab/string_path.rb | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index ad68a8ff2d6..8310564646e 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,15 +5,12 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO, better support for '../' and './' - # class StringPath attr_reader :path, :universe def initialize(path, universe, metadata = []) - @path = prepare(path) - @universe = universe.map { |entry| prepare(entry) } - @universe << './' unless @universe.include?('./') + @path = sanitize(path) + @universe = universe.map { |entry| sanitize(entry) } @metadata = metadata end @@ -60,15 +57,16 @@ module Gitlab def descendants return [] unless directory? - children = @universe.select { |entry| entry =~ /^#{@path}.+/ } - children.map { |path| new(path) } + select { |entry| entry =~ /^#{@path}.+/ } end def children return [] unless directory? return @children if @children - children = @universe.select { |entry| entry =~ %r{^#{@path}[^/]+/?$} } - @children = children.map { |path| new(path) } + + @children = select do |entry| + self.class.child?(@path, entry) + end end def directories @@ -104,9 +102,26 @@ module Gitlab self.class.new(path, @universe) end - def prepare(path) - return path if path =~ %r{^(/|\.|\.\.)} - path.dup.prepend('./') + def select + selected = @universe.select { |entry| yield entry } + selected.map { |path| new(path) } + end + + def sanitize(path) + self.class.sanitize(path) + end + + def self.sanitize(path) + # It looks like Pathname#new doesn't touch a file system, + # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe + + clean = Pathname.new(path).cleanpath.to_s + raise ArgumentError, 'Invalid path' if clean.start_with?('../') + clean + (path.end_with?('/') ? '/' : '') + end + + def self.child?(path, entry) + entry =~ %r{^#{path}[^/\s]+/?$} end end end -- cgit v1.2.1 From df41148662142ce20a77b092665f48dd4dfa7bfb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 2 Jan 2016 20:09:21 +0100 Subject: Improve path sanitization in `StringPath` --- lib/gitlab/string_path.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 8310564646e..e7d99a35869 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -57,7 +57,7 @@ module Gitlab def descendants return [] unless directory? - select { |entry| entry =~ /^#{@path}.+/ } + select { |entry| entry =~ /^#{Regexp.escape(@path)}.+/ } end def children @@ -65,7 +65,7 @@ module Gitlab return @children if @children @children = select do |entry| - self.class.child?(@path, entry) + entry =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?$} end end @@ -75,7 +75,7 @@ module Gitlab end def directories! - has_parent? ? directories.prepend(new(@path + '../')) : directories + has_parent? ? directories.prepend(parent) : directories end def files @@ -115,13 +115,12 @@ module Gitlab # It looks like Pathname#new doesn't touch a file system, # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe - clean = Pathname.new(path).cleanpath.to_s - raise ArgumentError, 'Invalid path' if clean.start_with?('../') - clean + (path.end_with?('/') ? '/' : '') - end + clean_path = Pathname.new(path).cleanpath.to_s + raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') - def self.child?(path, entry) - entry =~ %r{^#{path}[^/\s]+/?$} + prefix = './' unless clean_path =~ %r{^[\.|/]} + suffix = '/' if path.end_with?('/') || clean_path =~ /^[\.|\.\.]$/ + prefix.to_s + clean_path + suffix.to_s end end end -- cgit v1.2.1 From a7f99b67a0bf1160f41ebf4dc92c618eb13a7a10 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 13:08:49 +0100 Subject: Extract artifacts metadata implementation to separate class --- lib/gitlab/ci/build/artifacts/metadata.rb | 57 +++++++++++++++++++++++++++++++ lib/gitlab/string_path.rb | 4 +-- 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..5313182d55f --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,57 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + def initialize(file, path) + @file = file + + @path = path.sub(/^\.\//, '') + @path << '/' unless path.end_with?('/') + end + + def exists? + File.exists?(@file) + end + + def match! + raise StandardError, 'Metadata file not found !' unless exists? + paths, metadata = [], [] + + each do |line| + next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?\s} + + path, meta = line.split(' ') + paths.push(path) + metadata.push(meta) + end + + [paths, metadata.map { |meta| JSON.parse(meta) }] + end + + def to_string_path + universe, metadata = match! + ::Gitlab::StringPath.new(@path, universe, metadata) + end + + private + + def each + open do |file| + gzip = Zlib::GzipReader.new(file) + gzip.each_line { |line| yield line } + gzip.close + end + end + + def open + File.open(@file) { |file| yield file } + end + end + end + end + end +end diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index e7d99a35869..9948502e8ea 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -75,7 +75,7 @@ module Gitlab end def directories! - has_parent? ? directories.prepend(parent) : directories + @path =~ %r{^\./[^/]/} ? directories.prepend(parent) : directories end def files @@ -119,7 +119,7 @@ module Gitlab raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') prefix = './' unless clean_path =~ %r{^[\.|/]} - suffix = '/' if path.end_with?('/') || clean_path =~ /^[\.|\.\.]$/ + suffix = '/' if path.end_with?('/') || ['.', '..'].include?(clean_path) prefix.to_s + clean_path + suffix.to_s end end -- cgit v1.2.1 From f948c00757ca9529817c7368610b0c0d6734d48f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 13:40:42 +0100 Subject: Do not depend on universe when checking parent in `StringPath` --- lib/gitlab/string_path.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 9948502e8ea..4d024b3ff73 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -7,6 +7,7 @@ module Gitlab # class StringPath attr_reader :path, :universe + attr_accessor :name def initialize(path, universe, metadata = []) @path = sanitize(path) @@ -35,7 +36,7 @@ module Gitlab end def has_parent? - @universe.include?(@path.sub(basename, '')) + nodes > 1 end def parent @@ -48,7 +49,7 @@ module Gitlab end def name - @path.split(::File::SEPARATOR).last + @name || @path.split(::File::SEPARATOR).last end def has_descendants? @@ -75,7 +76,11 @@ module Gitlab end def directories! - @path =~ %r{^\./[^/]/} ? directories.prepend(parent) : directories + return directories unless has_parent? && directory? + + dotted_parent = parent + dotted_parent.name = '..' + directories.prepend(dotted_parent) end def files @@ -88,6 +93,10 @@ module Gitlab @metadata[index] end + def nodes + @path.count('/') + (file? ? 1 : 0) + end + def ==(other) @path == other.path && @universe == other.universe end -- cgit v1.2.1 From a5e1905d28e490fb4734bff0e02a1ecff4c7c029 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 14:00:49 +0100 Subject: Render 404 when artifacts path is invalid --- lib/gitlab/ci/build/artifacts/metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 5313182d55f..d5c3cd10e48 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -22,7 +22,7 @@ module Gitlab paths, metadata = [], [] each do |line| - next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?\s} + next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} path, meta = line.split(' ') paths.push(path) -- cgit v1.2.1 From cd3b8bbd2f8e7ad75a453441f83c46aeb1d37353 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 14:18:06 +0100 Subject: Add method that checks if path exists in `StringPath` --- lib/gitlab/string_path.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 4d024b3ff73..a6234d34e7d 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -19,6 +19,10 @@ module Gitlab @path end + def exists? + @path == './' || @universe.include?(@path) + end + def absolute? @path.start_with?('/') end -- cgit v1.2.1 From 1b1793c2530d7003d8baa5aa1912a4ab258d4a1c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 15:07:49 +0100 Subject: Show file size in artifacts browser using metadata --- lib/gitlab/ci/build/artifacts/metadata.rb | 5 ++--- lib/gitlab/string_path.rb | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d5c3cd10e48..1f3000e7c8a 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -23,13 +23,12 @@ module Gitlab each do |line| next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} - path, meta = line.split(' ') paths.push(path) metadata.push(meta) - end + end - [paths, metadata.map { |meta| JSON.parse(meta) }] + [paths, metadata.map { |meta| JSON.parse(meta, symbolize_names: true) }] end def to_string_path diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index a6234d34e7d..774d4244a2a 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -19,10 +19,6 @@ module Gitlab @path end - def exists? - @path == './' || @universe.include?(@path) - end - def absolute? @path.start_with?('/') end @@ -94,13 +90,17 @@ module Gitlab def metadata index = @universe.index(@path) - @metadata[index] + @metadata[index] || {} end def nodes @path.count('/') + (file? ? 1 : 0) end + def exists? + @path == './' || @universe.include?(@path) + end + def ==(other) @path == other.path && @universe == other.universe end @@ -112,7 +112,7 @@ module Gitlab private def new(path) - self.class.new(path, @universe) + self.class.new(path, @universe, @metadata) end def select -- cgit v1.2.1 From 387b27813d1d496c015f4f174812b4761c32648d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 8 Jan 2016 12:35:49 +0100 Subject: Change format of artifacts metadata from text to binary 0.0.1 This changes the format of metadata to handle paths, that may contain whitespace characters, new line characters and non-UTF-8 characters. Now those paths along with metadata in JSON format are stored as length-prefixed strings (uint32 prefix). Metadata file has a custom format: 1. First string field is metadata version field (string) 2. Second string field is metadata errors field (JSON strong) 3. All subsequent fields is pair of path (string) and path metadata in JSON format. Path's metadata contains all fields that where possible to extract from ZIP archive like date of modification, CRC, compressed size, uncompressed size and comment. --- lib/gitlab/ci/build/artifacts/metadata.rb | 68 ++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1f3000e7c8a..d90a64fdbb8 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -17,18 +17,33 @@ module Gitlab File.exists?(@file) end + def full_version + gzip do|gz| + read_string(gz) do |size| + raise StandardError, 'Artifacts metadata file empty!' unless size + end + end + end + + def version + full_version.match(/\w+ (\d+\.\d+\.\d+)/).captures.first + end + + def errors + gzip do|gz| + read_string(gz) # version + JSON.parse(read_string(gz)) + end + end + def match! raise StandardError, 'Metadata file not found !' unless exists? - paths, metadata = [], [] - each do |line| - next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} - path, meta = line.split(' ') - paths.push(path) - metadata.push(meta) + gzip do |gz| + read_string(gz) # version field + read_string(gz) # errors field + iterate_entries(gz) end - - [paths, metadata.map { |meta| JSON.parse(meta, symbolize_names: true) }] end def to_string_path @@ -38,11 +53,44 @@ module Gitlab private - def each + def iterate_entries(gz) + paths, metadata = [], [] + + until gz.eof? do + begin + path = read_string(gz) + meta = read_string(gz) + + next unless path =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + + paths.push(path) + metadata.push(JSON.parse(meta, symbolize_names: true)) + rescue JSON::ParserError + next + end + end + + [paths, metadata] + end + + def read_string_size(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_string_size(gz) + yield string_size if block_given? + return false unless string_size + gz.read(string_size).chomp + end + + def gzip open do |file| gzip = Zlib::GzipReader.new(file) - gzip.each_line { |line| yield line } + result = yield gzip gzip.close + result end end -- cgit v1.2.1 From 61fb47a43202332fe9ac57847996da929ba42d3f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 9 Jan 2016 14:41:43 +0100 Subject: Simplify implementation of build artifacts browser (refactoring) --- lib/gitlab/ci/build/artifacts/metadata.rb | 74 ++++++------- lib/gitlab/ci/build/artifacts/metadata/path.rb | 114 ++++++++++++++++++++ lib/gitlab/string_path.rb | 139 ------------------------- 3 files changed, 153 insertions(+), 174 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata/path.rb delete mode 100644 lib/gitlab/string_path.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d90a64fdbb8..996b5d91ff2 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,65 +6,54 @@ module Gitlab module Build module Artifacts class Metadata - def initialize(file, path) - @file = file - - @path = path.sub(/^\.\//, '') - @path << '/' unless path.end_with?('/') - end + VERSION_PATTERN = '[\w\s]+(\d+\.\d+\.\d+)' + attr_reader :file, :path, :full_version - def exists? - File.exists?(@file) - end - - def full_version - gzip do|gz| - read_string(gz) do |size| - raise StandardError, 'Artifacts metadata file empty!' unless size - end - end + def initialize(file, path) + @file, @path = file, path + @full_version = read_version + @path << '/' unless path.end_with?('/') || path.empty? end def version - full_version.match(/\w+ (\d+\.\d+\.\d+)/).captures.first + @full_version.match(/#{VERSION_PATTERN}/).captures.first end def errors gzip do|gz| read_string(gz) # version - JSON.parse(read_string(gz)) + errors = read_string(gz) + raise StandardError, 'Errors field not found!' unless errors + JSON.parse(errors) end end def match! - raise StandardError, 'Metadata file not found !' unless exists? - gzip do |gz| - read_string(gz) # version field - read_string(gz) # errors field - iterate_entries(gz) + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) end end - def to_string_path - universe, metadata = match! - ::Gitlab::StringPath.new(@path, universe, metadata) + def to_path + Path.new(@path, *match!) end private - def iterate_entries(gz) + def match_entries(gz) paths, metadata = [], [] - + child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + until gz.eof? do begin path = read_string(gz) meta = read_string(gz) - next unless path =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?$} - + next unless path =~ child_pattern + paths.push(path) - metadata.push(JSON.parse(meta, symbolize_names: true)) + metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) rescue JSON::ParserError next end @@ -73,16 +62,31 @@ module Gitlab [paths, metadata] end - def read_string_size(gz) + def read_version + gzip do|gz| + version_string = read_string(gz) + + unless version_string + raise StandardError, 'Artifacts metadata file empty!' + end + + unless version_string =~ /^#{VERSION_PATTERN}/ + raise StandardError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) binary = gz.read(4) binary.unpack('L>')[0] if binary end def read_string(gz) - string_size = read_string_size(gz) - yield string_size if block_given? + string_size = read_uint32(gz) return false unless string_size - gz.read(string_size).chomp + gz.read(string_size) end def gzip diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb new file mode 100644 index 00000000000..222903b348e --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -0,0 +1,114 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents a simplified path to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + class Path + attr_reader :path, :universe + attr_accessor :name + + def initialize(path, universe, metadata = []) + @path = path + @universe = universe + @metadata = metadata + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + end + + def directory? + @path.end_with?('/') || @path.blank? + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + new(@path.chomp(basename)) + end + + def basename + directory? ? name + ::File::SEPARATOR : name + end + + def name + @name || @path.split(::File::SEPARATOR).last + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]+/?$} + @children = select { |entry| entry =~ child_pattern } + end + + def directories + return [] unless directory? + children.select(&:directory?) + end + + def directories! + return directories unless has_parent? + + dotted_parent = parent + dotted_parent.name = '..' + directories.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @index ||= @universe.index(@path) + @metadata[@index] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def exists? + @path.blank? || @universe.include?(@path) + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @universe == other.universe + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def new(path) + self.class.new(path, @universe, @metadata) + end + + def select + selected = @universe.select { |entry| yield entry } + selected.map { |path| new(path) } + end + end + end + end +end diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb deleted file mode 100644 index 774d4244a2a..00000000000 --- a/lib/gitlab/string_path.rb +++ /dev/null @@ -1,139 +0,0 @@ -module Gitlab - ## - # Class that represents a simplified path to a file or directory - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - class StringPath - attr_reader :path, :universe - attr_accessor :name - - def initialize(path, universe, metadata = []) - @path = sanitize(path) - @universe = universe.map { |entry| sanitize(entry) } - @metadata = metadata - end - - def to_s - @path - end - - def absolute? - @path.start_with?('/') - end - - def relative? - !absolute? - end - - def directory? - @path.end_with?('/') - end - - def file? - !directory? - end - - def has_parent? - nodes > 1 - end - - def parent - return nil unless has_parent? - new(@path.sub(basename, '')) - end - - def basename - directory? ? name + ::File::SEPARATOR : name - end - - def name - @name || @path.split(::File::SEPARATOR).last - end - - def has_descendants? - descendants.any? - end - - def descendants - return [] unless directory? - select { |entry| entry =~ /^#{Regexp.escape(@path)}.+/ } - end - - def children - return [] unless directory? - return @children if @children - - @children = select do |entry| - entry =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?$} - end - end - - def directories - return [] unless directory? - children.select(&:directory?) - end - - def directories! - return directories unless has_parent? && directory? - - dotted_parent = parent - dotted_parent.name = '..' - directories.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - index = @universe.index(@path) - @metadata[index] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def exists? - @path == './' || @universe.include?(@path) - end - - def ==(other) - @path == other.path && @universe == other.universe - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def new(path) - self.class.new(path, @universe, @metadata) - end - - def select - selected = @universe.select { |entry| yield entry } - selected.map { |path| new(path) } - end - - def sanitize(path) - self.class.sanitize(path) - end - - def self.sanitize(path) - # It looks like Pathname#new doesn't touch a file system, - # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe - - clean_path = Pathname.new(path).cleanpath.to_s - raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') - - prefix = './' unless clean_path =~ %r{^[\.|/]} - suffix = '/' if path.end_with?('/') || ['.', '..'].include?(clean_path) - prefix.to_s + clean_path + suffix.to_s - end - end -end -- cgit v1.2.1 From 09a4a5aff8c53dd5930044ddbb285a95ef177d8a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 11 Jan 2016 09:57:03 +0100 Subject: Render only valid paths in artifacts metadata In this version we will support only relative paths in artifacts metadata. Support for absolute paths will be introduced later. --- lib/gitlab/ci/build/artifacts/metadata.rb | 10 +++++++--- lib/gitlab/ci/build/artifacts/metadata/path.rb | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 996b5d91ff2..91017f633a0 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -12,7 +12,6 @@ module Gitlab def initialize(file, path) @file, @path = file, path @full_version = read_version - @path << '/' unless path.end_with?('/') || path.empty? end def version @@ -43,14 +42,15 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] - child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} until gz.eof? do begin path = read_string(gz) meta = read_string(gz) - next unless path =~ child_pattern + next unless path =~ match_pattern + next unless path_valid?(path) paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) @@ -62,6 +62,10 @@ module Gitlab [paths, metadata] end + def path_valid?(path) + !(path.start_with?('/') || path =~ %r{\.?\./}) + end + def read_version gzip do|gz| version_string = read_string(gz) diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb index 222903b348e..80ead335d57 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -23,7 +23,7 @@ module Gitlab end def directory? - @path.end_with?('/') || @path.blank? + blank_node? || @path.end_with?('/') end def file? @@ -40,11 +40,11 @@ module Gitlab end def basename - directory? ? name + ::File::SEPARATOR : name + (directory? && !blank_node?) ? name + ::File::SEPARATOR : name end def name - @name || @path.split(::File::SEPARATOR).last + @name || @path.split(::File::SEPARATOR).last.to_s end def children @@ -83,7 +83,11 @@ module Gitlab end def exists? - @path.blank? || @universe.include?(@path) + blank_node? || @universe.include?(@path) + end + + def blank_node? + @path.empty? # "" is considered to be './' end def to_s -- cgit v1.2.1 From ffee05c242c87e004054b48747287c3160d1c19a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 11 Jan 2016 12:50:21 +0100 Subject: Improve invalid build artifacts metadata path matcher --- lib/gitlab/ci/build/artifacts/metadata.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 91017f633a0..2b17712cdbe 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -43,6 +43,7 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do begin @@ -50,7 +51,7 @@ module Gitlab meta = read_string(gz) next unless path =~ match_pattern - next unless path_valid?(path) + next if path =~ invalid_pattern paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) @@ -62,10 +63,6 @@ module Gitlab [paths, metadata] end - def path_valid?(path) - !(path.start_with?('/') || path =~ %r{\.?\./}) - end - def read_version gzip do|gz| version_string = read_string(gz) -- cgit v1.2.1 From 2be76355caa579d444c8e3c0d25563eb9778bfb2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 10:04:26 +0100 Subject: Support only valid UTF-8 paths in build artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 8 +++++--- lib/gitlab/ci/build/artifacts/metadata/path.rb | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 2b17712cdbe..d9c051be9f3 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -10,7 +10,8 @@ module Gitlab attr_reader :file, :path, :full_version def initialize(file, path) - @file, @path = file, path + @file = file + @path = path.force_encoding('ASCII-8BIT') @full_version = read_version end @@ -42,7 +43,7 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] - match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do @@ -51,11 +52,12 @@ module Gitlab meta = read_string(gz) next unless path =~ match_pattern + next unless path.force_encoding('UTF-8').valid_encoding? next if path =~ invalid_pattern paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) - rescue JSON::ParserError + rescue JSON::ParserError, Encoding::CompatibilityError next end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb index 80ead335d57..6896aa936d5 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -8,18 +8,24 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # + # This class is working only with UTF-8 encoded paths. + # class Path attr_reader :path, :universe attr_accessor :name def initialize(path, universe, metadata = []) - @path = path + @path = path.force_encoding('UTF-8') @universe = universe @metadata = metadata if path.include?("\0") raise ArgumentError, 'Path contains zero byte character!' end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end end def directory? @@ -51,7 +57,7 @@ module Gitlab return [] unless directory? return @children if @children - child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]+/?$} + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} @children = select { |entry| entry =~ child_pattern } end -- cgit v1.2.1 From 487b0a026f9efe2d8214c19a7b95b391708ba3f4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 11:02:15 +0100 Subject: Improvements, readability for artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d9c051be9f3..47efc51a76e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab next if path =~ invalid_pattern paths.push(path) - metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) + metadata.push(JSON.parse(meta, symbolize_names: true)) rescue JSON::ParserError, Encoding::CompatibilityError next end @@ -66,7 +66,7 @@ module Gitlab end def read_version - gzip do|gz| + gzip do |gz| version_string = read_string(gz) unless version_string @@ -95,9 +95,11 @@ module Gitlab def gzip open do |file| gzip = Zlib::GzipReader.new(file) - result = yield gzip - gzip.close - result + begin + yield gzip + ensure + gzip.close + end end end -- cgit v1.2.1 From e8995f9fd5c12882eafcf3766627f64a3d6f623d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 14:39:15 +0100 Subject: Modify artifacts upload API endpoint, add artifacts metadata --- lib/ci/api/builds.rb | 20 ++++++++++++++++---- lib/ci/api/entities.rb | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..2aaa06a022c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -78,11 +78,15 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file + # metadata (optional) - Artifacts metadata file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - real filename as send in Content-Disposition + # metadata.type - real content type as send in Content-Type # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -98,10 +102,17 @@ module Ci authenticate_build_token!(build) forbidden!('build is not running') unless build.running? - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file!(:file, artifacts_upload_path) + file_to_large! unless artifacts.size < max_artifacts_size + artifacts_attributes = { artifacts_file: artifacts } - if build.update_attributes(artifacts_file: file) + if params[:metadata] || params['metadata.path'.to_sym] + metadata = uploaded_file!(:metadata, artifacts_upload_path) + artifacts_attributes.store(:artifacts_metadata, metadata) + end + + if build.update_attributes(artifacts_attributes) present build, with: Entities::Build else render_validation_error!(build) @@ -148,6 +159,7 @@ module Ci not_found! unless build authenticate_build_token!(build) build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index e4ac0545ea2..dd34c661e25 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -31,6 +31,7 @@ module Ci expose :variables expose :artifacts_file, using: ArtifactFile + expose :artifacts_metadata, using: ArtifactFile end class Runner < Grape::Entity -- cgit v1.2.1 From 0b946029a1fb429db39fbec0cddccf40f7e2aa08 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 09:56:05 +0100 Subject: Update build artifacts API We do not want to allow runners to upload a metadata file. This needs to be generated by Workhorse only. --- lib/ci/api/builds.rb | 17 ++++++++--------- lib/ci/api/entities.rb | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2aaa06a022c..be2790571cb 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -79,14 +79,13 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file - # metadata (optional) - Artifacts metadata file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - real filename as send in Content-Disposition - # metadata.type - real content type as send in Content-Type + # metadata.name - filename (generated by Workhorse) + # metadata.type - content type (returned by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -101,19 +100,19 @@ module Ci not_found! unless build authenticate_build_token!(build) forbidden!('build is not running') unless build.running? + forbidden!('metadata reserved for workhorse') if params[:metadata] artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file!(:file, artifacts_upload_path) file_to_large! unless artifacts.size < max_artifacts_size - artifacts_attributes = { artifacts_file: artifacts } + build.artifacts_file = artifacts - if params[:metadata] || params['metadata.path'.to_sym] - metadata = uploaded_file!(:metadata, artifacts_upload_path) - artifacts_attributes.store(:artifacts_metadata, metadata) + if params[:'metadata.path'] && params[:'metadata.name'] + build.artifacts_metadata = uploaded_file!(:metadata, artifacts_upload_path) end - if build.update_attributes(artifacts_attributes) - present build, with: Entities::Build + if build.save + present(build, with: Entities::Build) else render_validation_error!(build) end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index dd34c661e25..e4ac0545ea2 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -31,7 +31,6 @@ module Ci expose :variables expose :artifacts_file, using: ArtifactFile - expose :artifacts_metadata, using: ArtifactFile end class Runner < Grape::Entity -- cgit v1.2.1 From a9783c439bd6e3322b6fc72371c9fe3837a63be5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 10:13:39 +0100 Subject: Make encoding of paths returned by metadata consistent (UTF-8) --- lib/gitlab/ci/build/artifacts/metadata.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 47efc51a76e..0c252c0bf30 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -20,7 +20,7 @@ module Gitlab end def errors - gzip do|gz| + gzip do |gz| read_string(gz) # version errors = read_string(gz) raise StandardError, 'Errors field not found!' unless errors @@ -36,7 +36,7 @@ module Gitlab end def to_path - Path.new(@path, *match!) + Path.new(@path.dup.force_encoding('UTF-8'), *match!) end private @@ -88,7 +88,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) - return false unless string_size + return nil unless string_size gz.read(string_size) end -- cgit v1.2.1 From 3f0c18f80e36561581ef6fa6dbfcec169e1a6e08 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 13:45:28 +0100 Subject: Simplify encoding related implementation in artifacts metadata --- lib/gitlab/ci/build/artifacts/metadata.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 0c252c0bf30..e9ec8f1302c 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -10,8 +10,7 @@ module Gitlab attr_reader :file, :path, :full_version def initialize(file, path) - @file = file - @path = path.force_encoding('ASCII-8BIT') + @file, @path = file, path @full_version = read_version end @@ -36,7 +35,7 @@ module Gitlab end def to_path - Path.new(@path.dup.force_encoding('UTF-8'), *match!) + Path.new(@path, *match!) end private @@ -48,11 +47,11 @@ module Gitlab until gz.eof? do begin - path = read_string(gz) - meta = read_string(gz) + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern - next unless path.force_encoding('UTF-8').valid_encoding? next if path =~ invalid_pattern paths.push(path) -- cgit v1.2.1 From 154b8ceba4ac2d92a2387ad50d7f2b4ed5b2dd8a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 14:02:36 +0100 Subject: Refactor build artifacts upload API endpoint --- lib/api/helpers.rb | 4 +++- lib/ci/api/builds.rb | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..d46b5c42967 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -289,12 +289,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index be2790571cb..fb87637b94f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -85,7 +85,6 @@ module Ci # file.type - real content type as send in Content-Type # metadata.path - path to locally stored body (generated by Workhorse) # metadata.name - filename (generated by Workhorse) - # metadata.type - content type (returned by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -99,17 +98,17 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? - forbidden!('metadata reserved for workhorse') if params[:metadata] + forbidden!('Build is not running!') unless build.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file!(:file, artifacts_upload_path) + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) + + bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - build.artifacts_file = artifacts - if params[:'metadata.path'] && params[:'metadata.name'] - build.artifacts_metadata = uploaded_file!(:metadata, artifacts_upload_path) - end + build.artifacts_file = artifacts + build.artifacts_metadata = metadata if build.save present(build, with: Entities::Build) -- cgit v1.2.1 From 6b0a43aff36f0bbb9050b3c04155a3ccd9c1a75b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 21:17:28 +0100 Subject: Improve readability of artifacts browser `Entry` related code --- lib/gitlab/ci/build/artifacts/metadata.rb | 5 +- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 125 ++++++++++++++++++++++++ lib/gitlab/ci/build/artifacts/metadata/path.rb | 124 ----------------------- 3 files changed, 128 insertions(+), 126 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata/entry.rb delete mode 100644 lib/gitlab/ci/build/artifacts/metadata/path.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index e9ec8f1302c..bfdfc9a1d7d 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -34,8 +34,9 @@ module Gitlab end end - def to_path - Path.new(@path, *match!) + def to_entry + entires, metadata = match! + Entry.new(@path, entires, metadata) end private diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..2fb6c327729 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,125 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entires + attr_accessor :name + + def initialize(path, entires, metadata = []) + @path = path.force_encoding('UTF-8') + @entires = entires + @metadata = metadata + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + new(@path.chomp(basename)) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entires { |entry| entry =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @index ||= @entires.index(@path) + @metadata[@index] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entires.include?(@path) + end + + def empty? + children.empty? + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entires == other.entires + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def new(path) + self.class.new(path, @entires, @metadata) + end + + def select_entires + selected = @entires.select { |entry| yield entry } + selected.map { |path| new(path) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb deleted file mode 100644 index 6896aa936d5..00000000000 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ /dev/null @@ -1,124 +0,0 @@ -module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents a simplified path to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Path - attr_reader :path, :universe - attr_accessor :name - - def initialize(path, universe, metadata = []) - @path = path.force_encoding('UTF-8') - @universe = universe - @metadata = metadata - - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' - end - - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' - end - end - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - new(@path.chomp(basename)) - end - - def basename - (directory? && !blank_node?) ? name + ::File::SEPARATOR : name - end - - def name - @name || @path.split(::File::SEPARATOR).last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select { |entry| entry =~ child_pattern } - end - - def directories - return [] unless directory? - children.select(&:directory?) - end - - def directories! - return directories unless has_parent? - - dotted_parent = parent - dotted_parent.name = '..' - directories.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @index ||= @universe.index(@path) - @metadata[@index] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def exists? - blank_node? || @universe.include?(@path) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @universe == other.universe - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def new(path) - self.class.new(path, @universe, @metadata) - end - - def select - selected = @universe.select { |entry| yield entry } - selected.map { |path| new(path) } - end - end - end - end -end -- cgit v1.2.1 From ad2b0358e0facd5c65c4141ce54c2e55bab165e6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 22:31:27 +0100 Subject: Improve readability of artifacts `Metadata` related code --- lib/gitlab/ci/build/artifacts/metadata.rb | 32 +++++++++---------------- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 26 ++++++++++---------- 2 files changed, 24 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index bfdfc9a1d7d..94821c0eae0 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,7 +6,9 @@ module Gitlab module Build module Artifacts class Metadata - VERSION_PATTERN = '[\w\s]+(\d+\.\d+\.\d+)' + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + attr_reader :file, :path, :full_version def initialize(file, path) @@ -15,7 +17,7 @@ module Gitlab end def version - @full_version.match(/#{VERSION_PATTERN}/).captures.first + @full_version.match(VERSION_PATTERN)[1] end def errors @@ -27,7 +29,7 @@ module Gitlab end end - def match! + def find_entries! gzip do |gz| 2.times { read_string(gz) } # version and errors fields match_entries(gz) @@ -35,8 +37,8 @@ module Gitlab end def to_entry - entires, metadata = match! - Entry.new(@path, entires, metadata) + entries, metadata = find_entries! + Entry.new(@path, entries, metadata) end private @@ -44,7 +46,6 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} - invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do begin @@ -53,7 +54,7 @@ module Gitlab next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern - next if path =~ invalid_pattern + next if path =~ INVALID_PATH_PATTERN paths.push(path) metadata.push(JSON.parse(meta, symbolize_names: true)) @@ -73,7 +74,7 @@ module Gitlab raise StandardError, 'Artifacts metadata file empty!' end - unless version_string =~ /^#{VERSION_PATTERN}/ + unless version_string =~ VERSION_PATTERN raise StandardError, 'Invalid version!' end @@ -92,19 +93,8 @@ module Gitlab gz.read(string_size) end - def gzip - open do |file| - gzip = Zlib::GzipReader.new(file) - begin - yield gzip - ensure - gzip.close - end - end - end - - def open - File.open(@file) { |file| yield file } + def gzip(&block) + Zlib::GzipReader.open(@file, &block) end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2fb6c327729..12bb1bf0346 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -11,12 +11,12 @@ module Gitlab # This class is working only with UTF-8 encoded paths. # class Entry - attr_reader :path, :entires + attr_reader :path, :entries attr_accessor :name - def initialize(path, entires, metadata = []) + def initialize(path, entries, metadata = []) @path = path.force_encoding('UTF-8') - @entires = entires + @entries = entries @metadata = metadata if path.include?("\0") @@ -42,7 +42,7 @@ module Gitlab def parent return nil unless has_parent? - new(@path.chomp(basename)) + new_entry(@path.chomp(basename)) end def basename @@ -58,7 +58,7 @@ module Gitlab return @children if @children child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entires { |entry| entry =~ child_pattern } + @children = select_entries { |entry| entry =~ child_pattern } end def directories(opts = {}) @@ -77,7 +77,7 @@ module Gitlab end def metadata - @index ||= @entires.index(@path) + @index ||= @entries.index(@path) @metadata[@index] || {} end @@ -90,7 +90,7 @@ module Gitlab end def exists? - blank_node? || @entires.include?(@path) + blank_node? || @entries.include?(@path) end def empty? @@ -102,7 +102,7 @@ module Gitlab end def ==(other) - @path == other.path && @entires == other.entires + @path == other.path && @entries == other.entries end def inspect @@ -111,13 +111,13 @@ module Gitlab private - def new(path) - self.class.new(path, @entires, @metadata) + def new_entry(path) + self.class.new(path, @entries, @metadata) end - def select_entires - selected = @entires.select { |entry| yield entry } - selected.map { |path| new(path) } + def select_entries + selected = @entries.select { |entry| yield entry } + selected.map { |path| new_entry(path) } end end end -- cgit v1.2.1 From 0d6e7b9d3d38e60e5a706956a853e7dc940e4574 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 23:24:28 +0100 Subject: Use Hash to store paths and entries metadata in artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 11 +++++------ lib/gitlab/ci/build/artifacts/metadata/entry.rb | 18 ++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 94821c0eae0..25ecf27d4e6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -37,14 +37,14 @@ module Gitlab end def to_entry - entries, metadata = find_entries! - Entry.new(@path, entries, metadata) + entries = find_entries! + Entry.new(@path, entries) end private def match_entries(gz) - paths, metadata = [], [] + entries = {} match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} until gz.eof? do @@ -56,14 +56,13 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - paths.push(path) - metadata.push(JSON.parse(meta, symbolize_names: true)) + entries.store(path, JSON.parse(meta, symbolize_names: true)) rescue JSON::ParserError, Encoding::CompatibilityError next end end - [paths, metadata] + entries end def read_version diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 12bb1bf0346..4dae02ce4f7 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -14,10 +14,9 @@ module Gitlab attr_reader :path, :entries attr_accessor :name - def initialize(path, entries, metadata = []) - @path = path.force_encoding('UTF-8') + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') @entries = entries - @metadata = metadata if path.include?("\0") raise ArgumentError, 'Path contains zero byte character!' @@ -42,7 +41,7 @@ module Gitlab def parent return nil unless has_parent? - new_entry(@path.chomp(basename)) + self.class.new(@path.chomp(basename), @entries) end def basename @@ -77,8 +76,7 @@ module Gitlab end def metadata - @index ||= @entries.index(@path) - @metadata[@index] || {} + @entries[@path] || {} end def nodes @@ -111,13 +109,9 @@ module Gitlab private - def new_entry(path) - self.class.new(path, @entries, @metadata) - end - def select_entries - selected = @entries.select { |entry| yield entry } - selected.map { |path| new_entry(path) } + selected = @entries.select { |entry, _metadata| yield entry } + selected.map { |path, _metadata| self.class.new(path, @entries) } end end end -- cgit v1.2.1 From be764a3a20c7cecce2a047ddd46aff954c33b306 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Jan 2016 12:12:05 +0100 Subject: Minor improvements in build arfifacts browser Added also a `Gitlab::Ci::Build::Artifacts::Metadata::ParserError` exception class. --- lib/gitlab/ci/build/artifacts/metadata.rb | 17 ++++++++++++----- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 25ecf27d4e6..1344f5d120b 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,6 +6,8 @@ module Gitlab module Build module Artifacts class Metadata + class ParserError < StandardError; end + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} @@ -24,8 +26,13 @@ module Gitlab gzip do |gz| read_string(gz) # version errors = read_string(gz) - raise StandardError, 'Errors field not found!' unless errors - JSON.parse(errors) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end end end @@ -56,7 +63,7 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - entries.store(path, JSON.parse(meta, symbolize_names: true)) + entries[path] = JSON.parse(meta, symbolize_names: true) rescue JSON::ParserError, Encoding::CompatibilityError next end @@ -70,11 +77,11 @@ module Gitlab version_string = read_string(gz) unless version_string - raise StandardError, 'Artifacts metadata file empty!' + raise ParserError, 'Artifacts metadata file empty!' end unless version_string =~ VERSION_PATTERN - raise StandardError, 'Invalid version!' + raise ParserError, 'Invalid version!' end version_string.chomp diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 4dae02ce4f7..25b71fc3275 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -57,7 +57,7 @@ module Gitlab return @children if @children child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |entry| entry =~ child_pattern } + @children = select_entries { |path| path =~ child_pattern } end def directories(opts = {}) @@ -110,7 +110,7 @@ module Gitlab private def select_entries - selected = @entries.select { |entry, _metadata| yield entry } + selected = @entries.select { |path, _metadata| yield path } selected.map { |path, _metadata| self.class.new(path, @entries) } end end -- cgit v1.2.1 From 05187f0fc671e1ddcbe6d7b0f76971d792acaada Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 13:09:36 +0100 Subject: Properly generate diff of orphan commits, like the first commit in a repository --- lib/gitlab/diff/parser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..516e59b87a3 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -56,8 +56,9 @@ module Gitlab private def filename?(line) - line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', - '--- /tmp/diffy', '+++ /tmp/diffy') + line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit + '--- /tmp/diffy', '+++ /tmp/diffy') end def identification_type(line) -- cgit v1.2.1 From c31d777c8f24029d7e11dd1e78eddf0c0b6e6f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 14 Jan 2016 14:20:21 +0100 Subject: Revert changes to how the notes are paginated in the API --- lib/api/notes.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index ebd9e97148c..174473f5371 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -22,11 +22,17 @@ module API @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) # We exclude notes that are cross-references and that cannot be viewed - # by the current user. + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. notes = - @noteable.notes. + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). reject { |n| n.cross_reference_not_visible_for?(current_user) } - present paginate(Kaminari.paginate_array(notes)), with: Entities::Note + present notes, with: Entities::Note end # Get a single +noteable+ note -- cgit v1.2.1 From 2c7d9cfa7dbd4e7716793bdb1ee9e081f13c33b2 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 14 Jan 2016 14:59:04 +0100 Subject: Move Ci::Build#available_statuses to AVAILABLE_STATUSES constant in CommitStatus --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 1337e1bb452..d293f988165 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -125,7 +125,7 @@ module API def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? - available_statuses = Ci::Build.available_statuses + available_statuses = ::CommitStatus::AVAILABLE_STATUSES scope = if scope.is_a?(String) [scope] -- cgit v1.2.1 From 78f5eb94fb15f7e4cc4208a4871b9533243bec40 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 6 Jan 2016 21:15:37 -0200 Subject: Import GitHub wiki into GitLab --- lib/gitlab/github_import/importer.rb | 13 +++++++++++++ lib/gitlab/github_import/project_creator.rb | 3 ++- lib/gitlab/github_import/wiki_formatter.rb | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/github_import/wiki_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2b0afbc7b39..f75227b4734 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -14,8 +16,11 @@ module Gitlab def execute import_issues import_pull_requests + import_wiki true + rescue Gitlab::Shell::Error + false end private @@ -66,6 +71,14 @@ module Gitlab noteable.notes.create!(comment.attributes) end end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6dd2b917c4a --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(".git", ".wiki.git") + end + end + end +end -- cgit v1.2.1 From a6a5990ee5f504107944c3bba5c18dbdea9f5207 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:05:18 -0200 Subject: Add Banzai::Filter::GollumTagsFilter for parsing Gollum's tags in HTML --- lib/banzai/filter/gollum_tags_filter.rb | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 lib/banzai/filter/gollum_tags_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb new file mode 100644 index 00000000000..f78b30bd4e9 --- /dev/null +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -0,0 +1,106 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + # HTML Filter for parsing Gollum's tags in HTML. + # + # Based on Gollum::Filter::Tags + # + # Context options: + # :project_wiki (required) - Current project wiki. + # + class GollumTagsFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match tag contents. + TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + + def call + search_text_nodes(doc).each do |node| + content = node.content + + next unless content.match(TAGS_PATTERN) + + html = process_tag($2) + + node.replace(html) if html != node.content + end + + doc + end + + private + + # Process a single tag into its final HTML form. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML version of the tag. + def process_tag(tag) + if html = process_image_tag(tag) + html + else + process_page_link_tag(tag) + end + end + + # Attempt to process the tag as an image tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid image tag or nil + # if it is not. + def process_image_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + name = parts[0].strip + + if file = project_wiki.find_file(name) + path = ::File.join project_wiki_base_path, file.path + elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i + path = name + end + + if path + content_tag(:img, nil, src: path) + end + end + + # Attempt to process the tag as a page link tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid page link tag or nil + # if it is not. + def process_page_link_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + if parts.size == 1 + url = parts[0].strip + else + name, url = *parts.compact.map(&:strip) + end + + content_tag(:a, name || url, href: url) + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + + # Ensure that a :project_wiki key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project_wiki + end + end + end +end -- cgit v1.2.1 From 4872b319c8152837bd36e7fc0a5ad912d1c3ef90 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:10:08 -0200 Subject: Use the WikiPipeline when rendering the wiki markdown content --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 +++++++++++++ lib/banzai/pipeline/wiki_pipeline.rb | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb create mode 100644 lib/banzai/pipeline/wiki_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb new file mode 100644 index 00000000000..1f98d3a183a --- /dev/null +++ b/lib/banzai/pipeline/gollum_tags_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class GollumTagsPipeline < BasePipeline + def self.filters + [ + Filter::GollumTagsFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb new file mode 100644 index 00000000000..4635a8d6471 --- /dev/null +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -0,0 +1,9 @@ +require 'banzai' + +module Banzai + module Pipeline + class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) + + end + end +end -- cgit v1.2.1 From 60f8434a9d7def80e43ca2e1c8e4e3e502856913 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 12:41:22 -0200 Subject: Refactoring Gitlab::GithubImport::Importer --- lib/gitlab/github_import/importer.rb | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index f75227b4734..18929b9113b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -14,13 +14,7 @@ module Gitlab end def execute - import_issues - import_pull_requests - import_wiki - - true - rescue Gitlab::Shell::Error - false + import_issues && import_pull_requests && import_wiki end private @@ -39,6 +33,10 @@ module Gitlab end end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_pull_requests @@ -53,6 +51,10 @@ module Gitlab import_comments_on_diff(pull_request.number, merge_request) end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_comments(issue_number, noteable) @@ -78,6 +80,10 @@ module Gitlab gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end + + true + rescue Gitlab::Shell::Error + false end end end -- cgit v1.2.1 From ca87bb652a1a8390cdc686059f670b89b274dce2 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:16:32 -0200 Subject: Remove GollumTagsPipeline --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 ------------- lib/banzai/pipeline/wiki_pipeline.rb | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb deleted file mode 100644 index 1f98d3a183a..00000000000 --- a/lib/banzai/pipeline/gollum_tags_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'banzai' - -module Banzai - module Pipeline - class GollumTagsPipeline < BasePipeline - def self.filters - [ - Filter::GollumTagsFilter - ] - end - end - end -end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 4635a8d6471..50b5450e70b 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -2,8 +2,10 @@ require 'banzai' module Banzai module Pipeline - class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) - + class WikiPipeline < FullPipeline + def self.filters + super.insert(1, Filter::GollumTagsFilter) + end end end end -- cgit v1.2.1 From 89e8b82b638e440bc487c4135cdfa4402fdffde5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:50:25 -0200 Subject: Make sure the .git is at the end on Gitlab::GithubImport::WikiFormatter --- lib/gitlab/github_import/wiki_formatter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6dd2b917c4a..6c592ff469c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -12,7 +12,7 @@ module Gitlab end def import_url - project.import_url.sub(".git", ".wiki.git") + project.import_url.sub(/\.git\z/, ".wiki.git") end end end -- cgit v1.2.1 From 765a2c73271cf311311c391e7e64f83e141c79ae Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 18:59:49 -0200 Subject: Refactoring Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 47 ++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f78b30bd4e9..e4a73030597 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -14,7 +14,10 @@ module Banzai include ActionView::Helpers::TagHelper # Pattern to match tag contents. - TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + TAGS_PATTERN = %r{\[\[(.+?)\]\]} + + # Pattern to match allowed image extensions + ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i def call search_text_nodes(doc).each do |node| @@ -22,9 +25,11 @@ module Banzai next unless content.match(TAGS_PATTERN) - html = process_tag($2) + html = process_tag($1) - node.replace(html) if html != node.content + if html && html != node.content + node.replace(html) + end end doc @@ -38,11 +43,11 @@ module Banzai # # Returns the String HTML version of the tag. def process_tag(tag) - if html = process_image_tag(tag) - html - else - process_page_link_tag(tag) - end + parts = tag.split('|') + + return if parts.size.zero? + + process_image_tag(parts) || process_page_link_tag(parts) end # Attempt to process the tag as an image tag. @@ -51,16 +56,15 @@ module Banzai # # Returns the String HTML if the tag is a valid image tag or nil # if it is not. - def process_image_tag(tag) - parts = tag.split('|') - return if parts.size.zero? + def process_image_tag(parts) + content = parts[0].strip - name = parts[0].strip + return unless image?(content) - if file = project_wiki.find_file(name) + if url?(content) + path = content + elsif file = project_wiki.find_file(content) path = ::File.join project_wiki_base_path, file.path - elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i - path = name end if path @@ -68,16 +72,21 @@ module Banzai end end + def image?(path) + path =~ ALLOWED_IMAGE_EXTENSIONS + end + + def url?(path) + path.start_with?(*%w(http https)) + end + # Attempt to process the tag as a page link tag. # # tag - The String tag contents (the stuff inside the double brackets). # # Returns the String HTML if the tag is a valid page link tag or nil # if it is not. - def process_page_link_tag(tag) - parts = tag.split('|') - return if parts.size.zero? - + def process_page_link_tag(parts) if parts.size == 1 url = parts[0].strip else -- cgit v1.2.1 From ac2c86055eb189690bf67cf97cc6eb5ec9c2be7b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 20:24:17 -0200 Subject: Update documentation on Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 40 +++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index e4a73030597..fe01dae4850 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -3,7 +3,28 @@ require 'html/pipeline/filter' module Banzai module Filter - # HTML Filter for parsing Gollum's tags in HTML. + # HTML Filter for parsing Gollum's tags in HTML. It's only parses the + # following tags: + # + # - Link to internal pages: + # + # * [[Bug Reports]] + # * [[How to Contribute|Contributing]] + # + # - Link to external resources: + # + # * [[http://en.wikipedia.org/wiki/Git_(software)]] + # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]] + # + # - Link internal images, the special attributes will be ignored: + # + # * [[images/logo.png]] + # * [[images/logo.png|alt=Logo]] + # + # - Link external images, the special attributes will be ignored: + # + # * [[http://example.com/images/logo.png]] + # * [[http://example.com/images/logo.png|alt=Logo]] # # Based on Gollum::Filter::Tags # @@ -13,7 +34,22 @@ module Banzai class GollumTagsFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper - # Pattern to match tag contents. + # Pattern to match tags content that should be parsed in HTML. + # + # Gollum's tags have been made to resemble the tags of other markups, + # especially MediaWiki. The basic syntax is: + # + # [[tag]] + # + # Some tags will accept attributes which are separated by pipe + # symbols.Some attributes must precede the tag and some must follow it: + # + # [[prefix-attribute|tag]] + # [[tag|suffix-attribute]] + # + # See https://github.com/gollum/gollum/wiki + # + # Rubular: http://rubular.com/r/7dQnE5CUCH TAGS_PATTERN = %r{\[\[(.+?)\]\]} # Pattern to match allowed image extensions -- cgit v1.2.1