From f4e7a8893d71bcbc92a5ecd16c95593f17d1ddf5 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 24 Dec 2015 19:05:57 +0100 Subject: Add builds API (listing, showing trace) --- lib/api/api.rb | 2 ++ lib/api/builds.rb | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/api/entities.rb | 13 +++++++++ 3 files changed, 95 insertions(+) create mode 100644 lib/api/builds.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7834262d612..266b5f48f8f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,5 +54,7 @@ module API mount Keys mount Tags mount Triggers + + mount Builds end end diff --git a/lib/api/builds.rb b/lib/api/builds.rb new file mode 100644 index 00000000000..ce4892b5eeb --- /dev/null +++ b/lib/api/builds.rb @@ -0,0 +1,80 @@ +module API + # Projects builds API + class Builds < Grape::API + before { authenticate! } + + resource :projects do + # Get a project repository commits + # + # Parameters: + # id (required) - The ID of a project + # scope (optional) - The scope of builds to show (one of: all, finished, running) + # page (optional) - The page number for pagination (default: 1) + # per_page (ooptional) - The value of items per page to show (default 30) + # Example Request: + # GET /projects/:id/builds/all + get ':id/builds' do + all_builds = user_project.builds + builds = all_builds.order('created_at DESC') + builds = + case params[:scope] + when 'all' + builds + when 'finished' + builds.finished + when 'running' + builds.running + when 'pending' + builds.pending + when 'success' + builds.success + when 'failed' + builds.failed + else + builds.running_or_pending.reverse_order + end + + page = (params[:page] || 1).to_i + per_page = (params[:per_page] || 30).to_i + + present builds.page(page).per(per_page), with: Entities::Build + end + + # Get a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/builds/:build_id + get ':id/builds/:build_id' do + present get_build(params[:build_id]), with: Entities::Build + end + + # Get a trace of a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/build/:build_id/trace + get ':id/builds/:build_id/trace' do + trace = get_build(params[:build_id]).trace + trace = + unless trace.nil? + trace.split("\n") + else + [] + end + + present trace + end + end + + helpers do + def get_build(id) + user_project.builds.where(id: id).first + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f8511ac5f5c..0bf50490eac 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -366,5 +366,18 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Build < Grape::Entity + expose :id + expose :status + expose :stage + expose :name + expose :ref + expose :commit + expose :runner + expose :created_at + expose :started_at + expose :finished_at + end end end -- cgit v1.2.1 From b5fef34f1e3beb60e7184cfb3420976bfa367137 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 24 Dec 2015 19:18:01 +0100 Subject: Fix example request url --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index ce4892b5eeb..0ddb9e98de6 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -12,7 +12,7 @@ module API # page (optional) - The page number for pagination (default: 1) # per_page (ooptional) - The value of items per page to show (default 30) # Example Request: - # GET /projects/:id/builds/all + # GET /projects/:id/builds get ':id/builds' do all_builds = user_project.builds builds = all_builds.order('created_at DESC') -- cgit v1.2.1 From f39959d00a1358ba7d73ebeaccb827738c8151ba Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 13:09:51 +0100 Subject: Add some fixes to builds API --- lib/api/builds.rb | 27 +++++++++------------------ lib/api/entities.rb | 20 ++++++++++++++++++-- 2 files changed, 27 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 0ddb9e98de6..863be0d5e40 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -15,23 +15,15 @@ module API # GET /projects/:id/builds get ':id/builds' do all_builds = user_project.builds - builds = all_builds.order('created_at DESC') + builds = all_builds.order('id DESC') builds = case params[:scope] - when 'all' - builds when 'finished' builds.finished when 'running' builds.running - when 'pending' - builds.pending - when 'success' - builds.success - when 'failed' - builds.failed else - builds.running_or_pending.reverse_order + builds end page = (params[:page] || 1).to_i @@ -59,15 +51,14 @@ module API # Example Request: # GET /projects/:id/build/:build_id/trace get ':id/builds/:build_id/trace' do - trace = get_build(params[:build_id]).trace - trace = - unless trace.nil? - trace.split("\n") - else - [] - end + build = get_build(params[:build_id]) + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary - present trace + trace = build.trace + body trace end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0bf50490eac..76b5d14f202 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,14 +367,30 @@ module API expose :id, :variables end + class CiCommit < Grape::Entity + expose :id + expose :ref + expose :sha + expose :committed_at + end + + class CiRunner < Grape::Entity + expose :id + expose :token + expose :description + expose :active + expose :is_shared + expose :name + end + class Build < Grape::Entity expose :id expose :status expose :stage expose :name expose :ref - expose :commit - expose :runner + expose :commit, with: CiCommit + expose :runner, with: CiRunner expose :created_at expose :started_at expose :finished_at -- cgit v1.2.1 From f4cff4dcd0b11d4597efe454731838fbf5803516 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 13:33:03 +0100 Subject: Modify build pagination to use 'paginate' helper --- lib/api/builds.rb | 5 +---- lib/api/helpers.rb | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 863be0d5e40..f219b0f5241 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -26,10 +26,7 @@ module API builds end - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 30).to_i - - present builds.page(page).per(per_page), with: Entities::Build + present paginate(builds), with: Entities::Build end # Get a specific build of a project diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..8fb5cd6ab63 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,8 +97,10 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) + page = (params[:page] || 1).to_i + per_page = (params[:per_page] || 30).to_i + + paginated = relation.page(page).per(per_page) add_pagination_headers(paginated, per_page) paginated -- cgit v1.2.1 From d398e78ea09c1f88800cd4c99f92dd8e5d1d0d39 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 15:49:13 +0100 Subject: Add endpoint for getting builds for a specific commit --- lib/api/builds.rb | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index f219b0f5241..7488863cdcf 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -4,7 +4,7 @@ module API before { authenticate! } resource :projects do - # Get a project repository commits + # Get a project builds # # Parameters: # id (required) - The ID of a project @@ -14,18 +14,21 @@ module API # Example Request: # GET /projects/:id/builds get ':id/builds' do - all_builds = user_project.builds - builds = all_builds.order('id DESC') - builds = - case params[:scope] - when 'finished' - builds.finished - when 'running' - builds.running - else - builds - end + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + present paginate(builds), with: Entities::Build + end + # GET builds for a specific commit of a project + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The SHA id of a commit + # Example Request: + # GET /projects/:id/builds/commit/:sha + get ':id/builds/commit/:sha' do + builds = user_project.ci_commits.find_by_sha(params[:sha]).builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build end @@ -63,6 +66,17 @@ module API def get_build(id) user_project.builds.where(id: id).first end + + def filter_builds(builds, scope) + case scope + when 'finished' + builds.finished + when 'running' + builds.running + else + builds + end + end end end end -- cgit v1.2.1 From e7d0746d9319119c459581615ae4205139bee444 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 16:38:02 +0100 Subject: Add 'not_found' notifications in build/trace details --- lib/api/builds.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 7488863cdcf..5ac24d0367a 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -19,7 +19,7 @@ module API present paginate(builds), with: Entities::Build end - # GET builds for a specific commit of a project + # Get builds for a specific commit of a project # # Parameters: # id (required) - The ID of a project @@ -40,7 +40,10 @@ module API # Example Request: # GET /projects/:id/builds/:build_id get ':id/builds/:build_id' do - present get_build(params[:build_id]), with: Entities::Build + build = get_build(params[:build_id]) + return not_found!(build) unless build + + present build, with: Entities::Build end # Get a trace of a specific build of a project @@ -52,6 +55,7 @@ module API # GET /projects/:id/build/:build_id/trace get ':id/builds/:build_id/trace' do build = get_build(params[:build_id]) + return not_found!(build) unless build header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' -- cgit v1.2.1 From 8d4555037a844d21dbf56c2995cff30782af920b Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 16:38:29 +0100 Subject: Add cancel/retry endpoints to build API --- lib/api/builds.rb | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 5ac24d0367a..16e4549d280 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -64,6 +64,42 @@ module API trace = build.trace body trace end + + # cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/cancel + post ':id/builds/:build_id/cancel' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + build.cancel + + present build, with: Entities::Build + end + + # cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/retry + post ':id/builds/:build_id/retry' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.retryable? + + build = Ci::Build.retry(build) + + present build, with: Entities::Build + end end helpers do @@ -81,6 +117,10 @@ module API builds end end + + def authorize_manage_builds! + authorize! :manage_builds, user_project + end end end end -- cgit v1.2.1 From b2fbeb377d8ba17352d40817b0b444196cb97636 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 28 Dec 2015 17:01:02 +0100 Subject: Remove changes in 'paginate' helper --- lib/api/helpers.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8fb5cd6ab63..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,10 +97,8 @@ module API end def paginate(relation) - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 30).to_i - - paginated = relation.page(page).per(per_page) + per_page = params[:per_page].to_i + paginated = relation.page(params[:page]).per(per_page) add_pagination_headers(paginated, per_page) paginated -- cgit v1.2.1 From 34657b821ae597de76ffd5a70d2b0b298dc270ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 15 Dec 2015 18:09:09 -0500 Subject: Add syntax highlighting to diff view. #3945 --- lib/rouge/lexers/gitlab_diff.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 lib/rouge/lexers/gitlab_diff.rb (limited to 'lib') diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb new file mode 100644 index 00000000000..e136d47df00 --- /dev/null +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -0,0 +1,20 @@ +Rouge::Token::Tokens.token(:InlineDiff, 'idiff') + +module Rouge + module Lexers + class GitlabDiff < RegexLexer + title "GitLab Diff" + tag 'gitlab_diff' + + state :root do + rule %r{(.*?)} do |match| + token InlineDiff, match[1] + end + + rule /(?:(?! Date: Wed, 23 Dec 2015 19:10:13 -0500 Subject: Set initial state on parent Lexer. #3945 --- lib/rouge/lexers/gitlab_diff.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index e136d47df00..c7aaeb92608 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -15,6 +15,10 @@ module Rouge delegate option(:parent_lexer) end end + + start do + option(:parent_lexer).reset! + end end end end -- cgit v1.2.1 From d2601211a0c7a44666501dea82a8488b08f8faa7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 29 Dec 2015 23:12:36 +0100 Subject: Add specs for build listings in API --- lib/api/builds.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 16e4549d280..a519224d2b8 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -27,7 +27,10 @@ module API # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do - builds = user_project.ci_commits.find_by_sha(params[:sha]).builds.order('id DESC') + commit = user_project.ci_commits.find_by_sha(params[:sha]) + return not_found! unless commit + + builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build end @@ -65,7 +68,7 @@ module API body trace end - # cancel a specific build of a project + # Cancel a specific build of a project # # parameters: # id (required) - the id of a project @@ -83,7 +86,7 @@ module API present build, with: Entities::Build end - # cancel a specific build of a project + # Retry a specific build of a project # # parameters: # id (required) - the id of a project -- cgit v1.2.1 From bb96d631537d3d8181f0d3b762603a012219c3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 00:52:50 -0500 Subject: New implementation for highlighting diff files. #3945 * It is more performant given now we process all the diff file instead of processing line by line. * Multiline comments are highlighted correctly. --- lib/gitlab/diff/file.rb | 4 +++ lib/gitlab/diff/highlight.rb | 55 +++++++++++++++++++++++++++++++++++++++++ lib/gitlab/diff/line.rb | 1 + lib/rouge/lexers/gitlab_diff.rb | 2 +- 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/diff/highlight.rb (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 79061cd0141..ff8765b8e26 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -15,6 +15,10 @@ module Gitlab @lines ||= parser.parse(raw_diff.lines) end + def highlighted_diff_lines + Gitlab::Diff::Highlight.process_diff_lines(self) + end + def mode_changed? !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb new file mode 100644 index 00000000000..f10b55eb00b --- /dev/null +++ b/lib/gitlab/diff/highlight.rb @@ -0,0 +1,55 @@ +module Gitlab + module Diff + class Highlight + def self.process_diff_lines(diff_file) + processor = new(diff_file) + processor.highlight + end + + def initialize(diff_file) + text_lines = diff_file.diff_lines.map(&:text) + @diff_file = diff_file + @diff_lines = diff_file.diff_lines + @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + @raw_lines = text_lines.join("\n") + end + + def highlight + @code = unescape_html(@raw_lines) + @highlighted_code = formatter.format(lexer.lex(@code)) + + update_diff_lines + end + + private + + def update_diff_lines + @highlighted_code.lines.each_with_index do |line, i| + @diff_lines[i].highlighted_text = "#{@diff_line_prefixes[i]}#{line}" + end + + @diff_lines + end + + def lexer + parent = Rouge::Lexer.guess(filename: @diff_file.new_path, source: @code).new rescue Rouge::Lexers::PlainText.new + Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) + end + + def unescape_html(content) + text = CGI.unescapeHTML(content) + text.gsub!(' ', ' ') + text + end + + def formatter + @formatter ||= Rouge::Formatters::HTMLGitlab.new( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + end + end + end +end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0072194606e..c48c69fb344 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,6 +2,7 @@ module Gitlab module Diff class Line attr_reader :type, :text, :index, :old_pos, :new_pos + attr_accessor :highlighted_text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index c7aaeb92608..d91dd6c4245 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -11,7 +11,7 @@ module Rouge token InlineDiff, match[1] end - rule /(?:(?! Date: Wed, 30 Dec 2015 13:10:28 -0500 Subject: Fix Rubocop complain. #3945 --- lib/gitlab/diff/highlight.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f10b55eb00b..adb437abed2 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -44,11 +44,11 @@ module Gitlab def formatter @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) end end end -- cgit v1.2.1 From 7de90f4b53f865dc417d022a9133372e57274549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 18:42:11 -0500 Subject: Fix broken spec and small refactor. #3945 --- lib/gitlab/diff/highlight.rb | 2 +- lib/gitlab/diff/line.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index adb437abed2..d0c2e3670c6 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -25,7 +25,7 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - @diff_lines[i].highlighted_text = "#{@diff_line_prefixes[i]}#{line}" + @diff_lines[i].text = "#{@diff_line_prefixes[i]}#{line}" end @diff_lines diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index c48c69fb344..03730b435ad 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,8 +1,8 @@ module Gitlab module Diff class Line - attr_reader :type, :text, :index, :old_pos, :new_pos - attr_accessor :highlighted_text + attr_reader :type, :index, :old_pos, :new_pos + attr_accessor :text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index -- cgit v1.2.1 From 8b079315d98a8ccf852592148632c6f052d9cb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 21:23:50 -0500 Subject: A bit of refactoring. #3945 --- lib/gitlab/diff/file.rb | 2 +- lib/gitlab/diff/highlight.rb | 16 ++++++++-------- lib/rouge/lexers/gitlab_diff.rb | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ff8765b8e26..69b38a32eeb 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -16,7 +16,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(self) + Gitlab::Diff::Highlight.process_diff_lines(new_path, diff_lines) end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index d0c2e3670c6..40a54ede2bb 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,15 +1,15 @@ module Gitlab module Diff class Highlight - def self.process_diff_lines(diff_file) - processor = new(diff_file) + def self.process_diff_lines(file_name, diff_lines) + processor = new(file_name, diff_lines) processor.highlight end - def initialize(diff_file) - text_lines = diff_file.diff_lines.map(&:text) - @diff_file = diff_file - @diff_lines = diff_file.diff_lines + def initialize(file_name, diff_lines) + text_lines = diff_lines.map(&:text) + @file_name = file_name + @diff_lines = diff_lines @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } @raw_lines = text_lines.join("\n") end @@ -32,7 +32,7 @@ module Gitlab end def lexer - parent = Rouge::Lexer.guess(filename: @diff_file.new_path, source: @code).new rescue Rouge::Lexers::PlainText.new + parent = Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) end @@ -43,7 +43,7 @@ module Gitlab end def formatter - @formatter ||= Rouge::Formatters::HTMLGitlab.new( + Rouge::Formatters::HTMLGitlab.new( nowrap: true, cssclass: 'code highlight', lineanchors: true, diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index d91dd6c4245..cbf272ee1de 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -2,6 +2,8 @@ Rouge::Token::Tokens.token(:InlineDiff, 'idiff') module Rouge module Lexers + # This new Lexer is required in order to avoid the inline diff markup + # to be tokenized, it will be rendered as raw HTML code if that happens. class GitlabDiff < RegexLexer title "GitLab Diff" tag 'gitlab_diff' -- cgit v1.2.1 From fd100e1ef1726418c81ab8833cf8bcf86fab6eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 21:44:12 -0500 Subject: Don't modify "match" diff lines. #3945 --- lib/gitlab/diff/highlight.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 40a54ede2bb..c780ea21775 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -25,7 +25,12 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - @diff_lines[i].text = "#{@diff_line_prefixes[i]}#{line}" + diff_line = @diff_lines[i] + + # ignore highlighting for "match" lines + next if diff_line.type == 'match' + + diff_line.text = "#{@diff_line_prefixes[i]}#{line}" end @diff_lines -- cgit v1.2.1 From 3fbcf52ec8decc3a4e331d52b2f47d7b85d399cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 31 Dec 2015 01:05:52 -0500 Subject: Apply syntax highlighting when expanding diff plus some refactor. #3945 --- lib/gitlab/diff/highlight.rb | 55 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index c780ea21775..7f340de65cc 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,31 +1,62 @@ module Gitlab module Diff class Highlight - def self.process_diff_lines(file_name, diff_lines) - processor = new(file_name, diff_lines) + # Apply syntax highlight to provided source code + # + # file_name - The file name related to the code. + # lines - It can be an Array of Gitlab::Diff::Line objects or simple Strings. + # When passing Strings you need to provide the required 'end of lines' + # chars ("\n") for each String given that we don't append them automatically. + # + # Returns an Array with the processed items. + def self.process_diff_lines(file_name, lines) + processor = new(file_name, lines) processor.highlight end - def initialize(file_name, diff_lines) - text_lines = diff_lines.map(&:text) - @file_name = file_name - @diff_lines = diff_lines - @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } - @raw_lines = text_lines.join("\n") + def initialize(file_name, lines) + @file_name = file_name + @lines = lines end def highlight - @code = unescape_html(@raw_lines) + return [] if @lines.empty? + + extract_line_prefixes + + @code = unescape_html(raw_content) @highlighted_code = formatter.format(lexer.lex(@code)) - update_diff_lines + is_diff_line? ? update_diff_lines : @highlighted_code.lines end private + def is_diff_line? + @lines.first.is_a?(Gitlab::Diff::Line) + end + + def text_lines + @text_lines ||= (is_diff_line? ? @lines.map(&:text) : @lines) + end + + def raw_content + @raw_content ||= text_lines.join(is_diff_line? ? "\n" : nil) + end + + def extract_line_prefixes + @diff_line_prefixes ||= begin + if is_diff_line? + text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + else + [] + end + end + end + def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - diff_line = @diff_lines[i] + diff_line = @lines[i] # ignore highlighting for "match" lines next if diff_line.type == 'match' @@ -33,7 +64,7 @@ module Gitlab diff_line.text = "#{@diff_line_prefixes[i]}#{line}" end - @diff_lines + @lines end def lexer -- cgit v1.2.1 From ea4777ff501e370a39ae30e76a955136afe3c1fa Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 15:19:13 +0100 Subject: Add features for list and show details of variables in API --- lib/api/api.rb | 2 ++ lib/api/entities.rb | 4 ++++ lib/api/variables.rb | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 lib/api/variables.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7834262d612..a9e1913f0f2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,5 +54,7 @@ module API mount Keys mount Tags mount Triggers + + mount Variables end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..f71d072f269 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -365,5 +365,9 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Variable < Grape::Entity + expose :id, :key, :value + end end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb new file mode 100644 index 00000000000..6517150f6f4 --- /dev/null +++ b/lib/api/variables.rb @@ -0,0 +1,43 @@ +module API + # Projects variables API + class Variables < Grape::API + before { authenticate! } + before { authorize_admin_project } + + resource :projects do + # Get project variables + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/variables + get ':id/variables' do + variables = user_project.variables + present paginate(variables), with: Entities::Variable + end + + # Get specifica bariable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated + # as ID other ways it's treated as `key` + # Example Reuest: + # GET /projects/:id/variables/:variable_id + get ':id/variables/:variable_id' do + variable_id = params[:variable_id] + variables = user_project.variables + variables = + if variable_id.match(/^\d+$/) + variables.where(id: variable_id.to_i) + else + variables.where(key: variable_id) + end + + present variables.first, with: Entities::Variable + end + end + end +end -- cgit v1.2.1 From a692ce1c079703c4f3947e1d0a29547189e94d0f Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 16:25:49 +0100 Subject: Add update feature for variables API --- lib/api/variables.rb | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 6517150f6f4..6522ecba70c 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -24,7 +24,7 @@ module API # id (required) - The ID of a project # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated # as ID other ways it's treated as `key` - # Example Reuest: + # Example Request: # GET /projects/:id/variables/:variable_id get ':id/variables/:variable_id' do variable_id = params[:variable_id] @@ -38,6 +38,25 @@ module API present variables.first, with: Entities::Variable end + + # Update existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID of a variable + # key (optional) - new value for `key` field of variable + # value (optional) - new value for `value` field of variable + # Example Request: + # PUT /projects/:id/variables/:variable_id + put ':id/variables/:variable_id' do + variable = user_project.variables.where(id: params[:variable_id].to_i).first + + variable.key = params[:key] + variable.value = params[:value] + variable.save! + + present variable, with: Entities::Variable + end end end end -- cgit v1.2.1 From 0d014feb1d216e692882976f0d70c3227eaec4ca Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 16:56:03 +0100 Subject: Add delete feature to variables API --- lib/api/variables.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 6522ecba70c..c70c7cd9d7b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -57,6 +57,18 @@ module API present variable, with: Entities::Variable end + + # Delete existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # variable_id (required) - The ID of a variable + # Exanoke Reqyest: + # DELETE /projects/:id/variables/:variable_id + delete ':id/variables/:variable_id' do + variable = user_project.variables.where(id: params[:variable_id].to_i).first + variable.destroy + end end end end -- cgit v1.2.1 From c5177dd5e2171b047a695802c979cf779522ba8a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 17:03:11 +0100 Subject: Add missing 'not_found' checks in variables API --- lib/api/variables.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index c70c7cd9d7b..dac2ba679c7 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -36,6 +36,8 @@ module API variables.where(key: variable_id) end + return not_found!('Variable') if variables.empty? + present variables.first, with: Entities::Variable end @@ -51,6 +53,8 @@ module API put ':id/variables/:variable_id' do variable = user_project.variables.where(id: params[:variable_id].to_i).first + return not_found!('Variable') unless variable + variable.key = params[:key] variable.value = params[:value] variable.save! @@ -67,6 +71,9 @@ module API # DELETE /projects/:id/variables/:variable_id delete ':id/variables/:variable_id' do variable = user_project.variables.where(id: params[:variable_id].to_i).first + + return not_found!('Variable') unless variable + variable.destroy end end -- cgit v1.2.1 From 937567b767e6d7b34dcaa1d9c83fc75464638683 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 22:30:07 +0100 Subject: Add create feature to variables API --- lib/api/variables.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index dac2ba679c7..fc63ac2f56a 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -41,6 +41,24 @@ module API present variables.first, with: Entities::Variable end + # Create a new variable in project + # + # Parameters: + # id (required) - The ID of a project + # key (required) - The key of variable being created + # value (required) - The value of variable being created + # Example Request: + # POST /projects/:id/variables + post ':id/variables' do + required_attributes! [:key, :value] + + variable = user_project.variables.create(key: params[:key], value: params[:value]) + return render_validation_error!(variable) unless variable.valid? + variable.save! + + present variable, with: Entities::Variable + end + # Update existing variable of a project # # Parameters: @@ -75,6 +93,8 @@ module API return not_found!('Variable') unless variable variable.destroy + + present variable, with: Entities::Variable end end end -- cgit v1.2.1 From 16bd4df083135e2e4a263b2e1bdd71b78a875ef7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 22:59:06 +0100 Subject: Fix a typo in method description --- lib/api/variables.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index fc63ac2f56a..b8bbcb6ce3b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -18,7 +18,7 @@ module API present paginate(variables), with: Entities::Variable end - # Get specifica bariable of a project + # Get specifica variable of a project # # Parameters: # id (required) - The ID of a project -- cgit v1.2.1 From 628297fe5692fc241c93ff34cece71132bfb9aed Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 31 Dec 2015 23:01:28 +0100 Subject: Remove incorrect 'default' values from method description --- lib/api/builds.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index a519224d2b8..92bf849824c 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -9,8 +9,8 @@ module API # Parameters: # id (required) - The ID of a project # scope (optional) - The scope of builds to show (one of: all, finished, running) - # page (optional) - The page number for pagination (default: 1) - # per_page (ooptional) - The value of items per page to show (default 30) + # page (optional) - The page number for pagination + # per_page (ooptional) - The value of items per page to show # Example Request: # GET /projects/:id/builds get ':id/builds' do -- cgit v1.2.1 From d9da81f736b770bb44c4869aef5d5c455e74ab7a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 4 Jan 2016 16:38:32 +0100 Subject: Add triggers feature to API --- lib/api/entities.rb | 10 ++++++++++ lib/api/triggers.rb | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..bc0cd76a2b8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -365,5 +365,15 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Trigger < Grape::Entity + expose :id, :token, :created_at, :updated_at, :deleted_at + expose :last_used do |repo_obj, _options| + if repo_obj.respond_to?(:last_trigger_request) + request = repo_obj.last_trigger_request + request.created_at if request + end + end + end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 2781f1cf191..9a1e3fdc976 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -43,6 +43,24 @@ module API render_api_error!(errors, 400) end end + + # Get triggers list + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/triggers + get ':id/triggers' do + authenticate! + authorize_admin_project + + triggers = user_project.triggers.includes(:trigger_requests) + triggers = paginate(triggers) + + present triggers, with: Entities::Trigger + end end end end -- cgit v1.2.1 From f00607431cd13a952731e36701ebc3b39e64d09b Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 11:27:38 +0100 Subject: Add delete feature to triggers API --- lib/api/triggers.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 9a1e3fdc976..3cb7810241f 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -61,6 +61,25 @@ module API present triggers, with: Entities::Trigger end + + # Delete trigger + # + # Parameters: + # id (required) - The ID of a project + # trigger_id - The ID of trigger to delete + # Example Request: + # DELETE /projects/:id/triggers/:trigger_id + delete ':id/triggers/:trigger_id' do + authenticate! + authorize_admin_project + + trigger = user_project.triggers.where(id: params[:trigger_id].to_i).first + return not_found!('Trigger') unless trigger + + trigger.destroy + + present trigger, with: Entities::Trigger + end end end end -- cgit v1.2.1 From 49c8bf4e9b510be51859dcc301cb46b29b750cb0 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 11:44:10 +0100 Subject: Add create feature to triggers API --- lib/api/triggers.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 3cb7810241f..38cf1e9a2e0 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,6 +62,22 @@ module API present triggers, with: Entities::Trigger end + # Create trigger + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/triggers + post ':id/triggers' do + authenticate! + authorize_admin_project + + trigger = user_project.triggers.new + trigger.save + + present trigger, with: Entities::Trigger + end + # Delete trigger # # Parameters: -- cgit v1.2.1 From 8675664655c4e0f1e043afa88ff1fd75ae5a6a9e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 12:25:16 +0100 Subject: Get show details feature to triggers API --- lib/api/triggers.rb | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 38cf1e9a2e0..0e548b936cd 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,6 +62,32 @@ module API present triggers, with: Entities::Trigger end + # Get specific trigger of a project + # + # Parameters: + # id (required) - The ID of a project + # trigger_id (required) - The ID or `token` of a trigger to show; if trigger_id contains only digits it's + # treated as ID other ways it's reated as `key` + # Example Request: + # GET /projects/:id/triggers/:trigger_id + get ':id/triggers/:trigger_id' do + authenticate! + authorize_admin_project + + trigger_id = params[:trigger_id] + triggers = user_project.triggers + triggers = + if trigger_id.match(/^\d+$/) + triggers.where(id: trigger_id.to_i) + else + triggers.where(token: trigger_id) + end + + return not_found!('Trigger') if triggers.empty? + + present triggers.first, with: Entities::Trigger + end + # Create trigger # # Parameters: -- cgit v1.2.1 From f60bceb988bd629f9adecc070ef5579d264f27c6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 13:20:06 +0100 Subject: Add CI data to projcet entity --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..e8154e0f383 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,6 +71,9 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + + expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex + expose :runners_token end class ProjectMember < UserBasic -- cgit v1.2.1 From 0a21731e3bc400ceb9898c9efbc2a186f5348e09 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 15:00:25 +0100 Subject: Add ci fields in project create/update feature API --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..563c12e4f74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) + if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0781236cf6d..7dd6b133f9b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,10 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -159,7 +162,10 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -215,7 +221,10 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :build_allow_git_fetch, + :build_timeout, + :build_coverage_regex] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? -- cgit v1.2.1 From a862ade55bf68f56734538b40e02e56036f8a1bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 5 Jan 2016 16:36:14 +0100 Subject: Update ./doc/api/ --- lib/api/builds.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 92bf849824c..6b0edcff820 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -8,7 +8,7 @@ module API # # Parameters: # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one of: all, finished, running) + # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) # page (optional) - The page number for pagination # per_page (ooptional) - The value of items per page to show # Example Request: @@ -24,6 +24,7 @@ module API # Parameters: # id (required) - The ID of a project # sha (required) - The SHA id of a commit + # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do -- cgit v1.2.1 From ba9799b42f5bc861df1fc5d41c149cf72e9daf04 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 10:39:02 +0100 Subject: Update ./doc/api --- lib/api/projects.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dd6b133f9b..31b081266a8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -98,6 +98,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -146,6 +149,9 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -207,6 +213,9 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # build_allow_git_fetch (optional) + # build_timeout (optional) + # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do -- cgit v1.2.1 From b60c146267dfa8dc1c170426e1817c6b2a168d1a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 13:49:38 +0100 Subject: Change :variable_id to :key as resource ID in API --- lib/api/variables.rb | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index b8bbcb6ce3b..cc038e5731d 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -22,19 +22,12 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID OR `key` of variable to show; if variable_id contains only digits it's treated - # as ID other ways it's treated as `key` + # key (required) - The `key` of variable # Example Request: - # GET /projects/:id/variables/:variable_id - get ':id/variables/:variable_id' do - variable_id = params[:variable_id] - variables = user_project.variables - variables = - if variable_id.match(/^\d+$/) - variables.where(id: variable_id.to_i) - else - variables.where(key: variable_id) - end + # GET /projects/:id/variables/:key + get ':id/variables/:key' do + key = params[:key] + variables = user_project.variables.where(key: key) return not_found!('Variable') if variables.empty? @@ -45,8 +38,8 @@ module API # # Parameters: # id (required) - The ID of a project - # key (required) - The key of variable being created - # value (required) - The value of variable being created + # key (required) - The key of variable + # value (required) - The value of variable # Example Request: # POST /projects/:id/variables post ':id/variables' do @@ -63,17 +56,15 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID of a variable - # key (optional) - new value for `key` field of variable - # value (optional) - new value for `value` field of variable + # key (optional) - The `key` of variable + # value (optional) - New value for `value` field of variable # Example Request: - # PUT /projects/:id/variables/:variable_id - put ':id/variables/:variable_id' do - variable = user_project.variables.where(id: params[:variable_id].to_i).first + # PUT /projects/:id/variables/:key + put ':id/variables/:key' do + variable = user_project.variables.where(key: params[:key]).first return not_found!('Variable') unless variable - variable.key = params[:key] variable.value = params[:value] variable.save! @@ -84,11 +75,11 @@ module API # # Parameters: # id (required) - The ID of a project - # variable_id (required) - The ID of a variable + # key (required) - The ID of a variable # Exanoke Reqyest: - # DELETE /projects/:id/variables/:variable_id - delete ':id/variables/:variable_id' do - variable = user_project.variables.where(id: params[:variable_id].to_i).first + # DELETE /projects/:id/variables/:key + delete ':id/variables/:key' do + variable = user_project.variables.where(key: params[:key]).first return not_found!('Variable') unless variable -- cgit v1.2.1 From b60445906849e84ff52ac6a5d7d501bb5a21eb60 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 14:10:49 +0100 Subject: Update ./doc/api --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f71d072f269..db3164d9d9c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,7 +367,7 @@ module API end class Variable < Grape::Entity - expose :id, :key, :value + expose :key, :value end end end -- cgit v1.2.1 From e0ec69d919cb44194e76034f2324ec0d4f5f1df6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 7 Jan 2016 18:48:33 +0100 Subject: Change 'trigger_id' to 'token' as resource ID in triggers API --- lib/api/entities.rb | 2 +- lib/api/triggers.rb | 25 ++++++++----------------- 2 files changed, 9 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index bc0cd76a2b8..37c483b45ec 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,7 +367,7 @@ module API end class Trigger < Grape::Entity - expose :id, :token, :created_at, :updated_at, :deleted_at + expose :token, :created_at, :updated_at, :deleted_at expose :last_used do |repo_obj, _options| if repo_obj.respond_to?(:last_trigger_request) request = repo_obj.last_trigger_request diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 0e548b936cd..25bb8aef20b 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -66,23 +66,14 @@ module API # # Parameters: # id (required) - The ID of a project - # trigger_id (required) - The ID or `token` of a trigger to show; if trigger_id contains only digits it's - # treated as ID other ways it's reated as `key` + # token (required) - The `token` of a trigger # Example Request: - # GET /projects/:id/triggers/:trigger_id - get ':id/triggers/:trigger_id' do + # GET /projects/:id/triggers/:token + get ':id/triggers/:token' do authenticate! authorize_admin_project - trigger_id = params[:trigger_id] - triggers = user_project.triggers - triggers = - if trigger_id.match(/^\d+$/) - triggers.where(id: trigger_id.to_i) - else - triggers.where(token: trigger_id) - end - + triggers = user_project.triggers.where(token: params[:token]) return not_found!('Trigger') if triggers.empty? present triggers.first, with: Entities::Trigger @@ -108,14 +99,14 @@ module API # # Parameters: # id (required) - The ID of a project - # trigger_id - The ID of trigger to delete + # token (required) - The `token` of a trigger # Example Request: - # DELETE /projects/:id/triggers/:trigger_id - delete ':id/triggers/:trigger_id' do + # DELETE /projects/:id/triggers/:token + delete ':id/triggers/:token' do authenticate! authorize_admin_project - trigger = user_project.triggers.where(id: params[:trigger_id].to_i).first + trigger = user_project.triggers.where(token: params[:token]).first return not_found!('Trigger') unless trigger trigger.destroy -- cgit v1.2.1 From f1f4fdf778245cab74ff9cda2a421315c21a99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 7 Jan 2016 21:08:57 -0500 Subject: Don't process inline diffs on backend. #3945 --- lib/gitlab/diff/parser.rb | 5 +-- lib/gitlab/inline_diff.rb | 104 ---------------------------------------------- 2 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 lib/gitlab/inline_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..177bad4b1cf 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -11,13 +11,10 @@ module Gitlab line_new = 1 type = nil - lines_arr = ::Gitlab::InlineDiff.processing lines - - lines_arr.each do |line| + @lines.each do |line| next if filename?(line) full_line = html_escape(line.gsub(/\n/, '')) - full_line = ::Gitlab::InlineDiff.replace_markers full_line if line.match(/^@@ -/) type = "match" diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb deleted file mode 100644 index 44507bde25d..00000000000 --- a/lib/gitlab/inline_diff.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Gitlab - class InlineDiff - class << self - - START = "#!idiff-start!#" - FINISH = "#!idiff-finish!#" - - def processing(diff_arr) - indexes = _indexes_of_changed_lines diff_arr - - indexes.each do |index| - first_line = diff_arr[index+1] - second_line = diff_arr[index+2] - - # Skip inline diff if empty line was replaced with content - next if first_line == "-\n" - - first_token = find_first_token(first_line, second_line) - apply_first_token(diff_arr, index, first_token) - - last_token = find_last_token(first_line, second_line, first_token) - apply_last_token(diff_arr, index, last_token) - end - - diff_arr - end - - def apply_first_token(diff_arr, index, first_token) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end - end - - def apply_last_token(diff_arr, index, last_token) - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - end - - def find_first_token(first_line, second_line) - max_length = [first_line.size, second_line.size].max - first_the_same_symbols = 0 - - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_line[0..first_the_same_symbols][1..-1] - end - - def find_last_token(first_line, second_line, first_token) - max_length = [first_line.size, second_line.size].max - last_the_same_symbols = 0 - - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - - if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - - last_the_same_symbols += 1 - first_line[last_the_same_symbols..-1] - end - - def _indexes_of_changed_lines(diff_arr) - chain_of_first_symbols = "" - diff_arr.each_with_index do |line, i| - chain_of_first_symbols += line[0] - end - chain_of_first_symbols.gsub!(/[^\-\+]/, "#") - - offset = 0 - indexes = [] - while index = chain_of_first_symbols.index("#-+#", offset) - indexes << index - offset = index + 1 - end - indexes - end - - def replace_markers(line) - line.gsub!(START, "") - line.gsub!(FINISH, "") - line - end - end - end -end -- cgit v1.2.1 From 21b602c60ad787b63039d804a5e15b43d0d3c32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 7 Jan 2016 22:37:01 -0500 Subject: Change strategy to highlight diffs. #3945 Now we apply syntax highlighting to the whole old and new files. This basically help us to highlight adequately multiline content. --- lib/gitlab/diff/file.rb | 8 +++-- lib/gitlab/diff/highlight.rb | 70 +++++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 69b38a32eeb..cb93c6a574a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,13 +1,15 @@ module Gitlab module Diff class File - attr_reader :diff + attr_reader :diff, :repository, :new_ref, :old_ref delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff) + def initialize(diff, diff_refs, repository) @diff = diff + @repository = repository + @old_ref, @new_ref = diff_refs end # Array of Gitlab::DIff::Line objects @@ -16,7 +18,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(new_path, diff_lines) + Gitlab::Diff::Highlight.process_diff_lines(self) end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7f340de65cc..0d0a3268107 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,22 +1,43 @@ module Gitlab module Diff class Highlight + attr_reader :diff_file + + delegate :repository, :old_path, :new_path, :old_ref, :new_ref, + to: :diff_file, prefix: :diff + # Apply syntax highlight to provided source code # - # file_name - The file name related to the code. - # lines - It can be an Array of Gitlab::Diff::Line objects or simple Strings. - # When passing Strings you need to provide the required 'end of lines' - # chars ("\n") for each String given that we don't append them automatically. + # diff_file - an instance of Gitlab::Diff::File # # Returns an Array with the processed items. - def self.process_diff_lines(file_name, lines) - processor = new(file_name, lines) + def self.process_diff_lines(diff_file) + processor = new(diff_file) processor.highlight end - def initialize(file_name, lines) - @file_name = file_name - @lines = lines + def self.process_file(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + content = blob.data + lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new + formatter.format(lexer.lex(content)).lines + end + + def self.formatter + @formatter ||= Rouge::Formatters::HTMLGitlab.new( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + end + + def initialize(diff_file) + @diff_file = diff_file + @file_name = diff_file.new_path + @lines = diff_file.diff_lines end def highlight @@ -47,7 +68,7 @@ module Gitlab def extract_line_prefixes @diff_line_prefixes ||= begin if is_diff_line? - text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } else [] end @@ -57,11 +78,17 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| diff_line = @lines[i] + line_prefix = @diff_line_prefixes[i] || ' ' # ignore highlighting for "match" lines next if diff_line.type == 'match' - diff_line.text = "#{@diff_line_prefixes[i]}#{line}" + case diff_line.type + when 'new', nil + diff_line.text = new_lines[diff_line.new_pos - 1].try(:gsub!, /\A\s/, line_prefix) + when 'old' + diff_line.text = old_lines[diff_line.old_pos - 1].try(:gsub!, /\A\s/, line_prefix) + end end @lines @@ -79,12 +106,21 @@ module Gitlab end def formatter - Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + self.class.formatter + end + + def old_lines + @old_lines ||= begin + lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) + lines.map! { |line| " #{line}" } + end + end + + def new_lines + @new_lines ||= begin + lines = self.class.process_file(diff_repository, diff_new_ref, diff_new_path) + lines.map! { |line| " #{line}" } + end end end end -- cgit v1.2.1 From 549a2fa7873366b52e9ba3caa849073b7b958b73 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 8 Jan 2016 14:01:31 +0100 Subject: Modify builds scope filtering in builds API --- lib/api/builds.rb | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 6b0edcff820..6aae1856953 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -8,9 +8,8 @@ module API # # Parameters: # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) - # page (optional) - The page number for pagination - # per_page (ooptional) - The value of items per page to show + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) # Example Request: # GET /projects/:id/builds get ':id/builds' do @@ -24,7 +23,8 @@ module API # Parameters: # id (required) - The ID of a project # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one of: all, finished, running; default: all) + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) # Example Request: # GET /projects/:id/builds/commit/:sha get ':id/builds/commit/:sha' do @@ -112,14 +112,21 @@ module API end def filter_builds(builds, scope) - case scope - when 'finished' - builds.finished - when 'running' - builds.running - else - builds - end + available_scopes = Ci::Build.available_statuses + scope = + if scope.is_a?(String) || scope.is_a?(Symbol) + available_scopes & [scope.to_s] + elsif scope.is_a?(Array) + available_scopes & scope + elsif scope.respond_to?(:to_h) + available_scopes & scope.to_h.values + else + [] + end + + return builds if scope.empty? + + builds.where(status: scope) end def authorize_manage_builds! -- cgit v1.2.1 From 6e7db8e23e169bcbf0847ece27b9e44e00ae572b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 30 Dec 2015 16:52:02 -0200 Subject: Prevent ldap_blocked users from being blocked/unblocked by the API --- lib/api/users.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 0d7813428e2..01fd90139b0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -284,10 +284,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user + if !user + not_found!('User') + elsif !user.ldap_blocked? user.block else - not_found!('User') + forbidden!('LDAP blocked users cannot be modified by the API') end end @@ -299,10 +301,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user + if !user + not_found!('User') + elsif !user.ldap_blocked? user.activate else - not_found!('User') + forbidden!('LDAP blocked users cannot be unblocked by the API') end end end -- cgit v1.2.1 From d6dc088affeee4568e771e1d7894e0bcdb955af8 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 30 Dec 2015 20:56:26 -0200 Subject: LDAP synchronization block/unblock new states --- lib/gitlab/ldap/access.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..76cb48d7aa6 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -37,15 +37,15 @@ module Gitlab # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block + user.ldap_block false else - user.activate if user.blocked? && !ldap_config.block_auto_created_users + user.activate if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.block + user.ldap_block false end rescue -- cgit v1.2.1 From 47e4613f4adc2d6ef4b066a87ec772ef8044bdd5 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 7 Jan 2016 14:01:01 -0200 Subject: Code style fixes and some code simplified --- lib/gitlab/ldap/access.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 76cb48d7aa6..ebd9260ad5d 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -40,7 +40,9 @@ module Gitlab user.ldap_block false else - user.activate if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? + if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? + user.activate + end true end else -- cgit v1.2.1 From 6282202ee84f80f2197698b6f132abdf588e94d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Fri, 8 Jan 2016 15:17:45 -0500 Subject: Remove custom Lexer. #3945 [ci skip] Inline diff is going to be generated client side now. #3945 --- lib/gitlab/diff/highlight.rb | 3 +-- lib/rouge/lexers/gitlab_diff.rb | 26 -------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 lib/rouge/lexers/gitlab_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0d0a3268107..7dd44b6004a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -95,8 +95,7 @@ module Gitlab end def lexer - parent = Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new - Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) + Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new end def unescape_html(content) diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb deleted file mode 100644 index cbf272ee1de..00000000000 --- a/lib/rouge/lexers/gitlab_diff.rb +++ /dev/null @@ -1,26 +0,0 @@ -Rouge::Token::Tokens.token(:InlineDiff, 'idiff') - -module Rouge - module Lexers - # This new Lexer is required in order to avoid the inline diff markup - # to be tokenized, it will be rendered as raw HTML code if that happens. - class GitlabDiff < RegexLexer - title "GitLab Diff" - tag 'gitlab_diff' - - state :root do - rule %r{(.*?)} do |match| - token InlineDiff, match[1] - end - - rule /(?:(?! Date: Fri, 8 Jan 2016 22:57:42 +0100 Subject: Modify entities for builds API --- lib/api/builds.rb | 16 +++++++++++----- lib/api/entities.rb | 37 ++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 6aae1856953..33e6ed24101 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -15,7 +15,9 @@ module API get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get builds for a specific commit of a project @@ -33,7 +35,8 @@ module API builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get a specific build of a project @@ -47,7 +50,8 @@ module API build = get_build(params[:build_id]) return not_found!(build) unless build - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Get a trace of a specific build of a project @@ -84,7 +88,8 @@ module API build.cancel - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end # Retry a specific build of a project @@ -102,7 +107,8 @@ module API build = Ci::Build.retry(build) - present build, with: Entities::Build + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f21da54b8fc..cb00b392db9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -366,16 +366,8 @@ module API expose :id, :variables end - class CiCommit < Grape::Entity + class Runner < Grape::Entity expose :id - expose :ref - expose :sha - expose :committed_at - end - - class CiRunner < Grape::Entity - expose :id - expose :token expose :description expose :active expose :is_shared @@ -383,16 +375,23 @@ module API end class Build < Grape::Entity - expose :id - expose :status - expose :stage - expose :name - expose :ref - expose :commit, with: CiCommit - expose :runner, with: CiRunner - expose :created_at - expose :started_at - expose :finished_at + expose :id, :status, :stage, :name, :ref, :tag, :coverage, :user + expose :created_at, :started_at, :finished_at + expose :download_url do |repo_obj, options| + if options[:user_can_download_artifacts] + repo_obj.download_url + else + nil + end + end + expose :commit, with: RepoCommit do |repo_obj, _options| + if repo_obj.respond_to?(:commit) + repo_obj.commit.commit_data + else + nil + end + end + expose :runner, with: Runner end end end -- cgit v1.2.1 From 4eb27d7c72d57015c7551a00e34a54cefc2d3db9 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 8 Jan 2016 23:33:45 +0100 Subject: Add some modifications to builds API and specs --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 33e6ed24101..05d1b8d92ea 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -103,7 +103,7 @@ module API authorize_manage_builds! build = get_build(params[:build_id]) - return not_found!(build) unless build && build.retryable? + return forbidden!('Build is not retryable') unless build && build.retryable? build = Ci::Build.retry(build) -- cgit v1.2.1 From 78d7c0e0d81c40e0d9541e4ef0dd15df30d457e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Fri, 8 Jan 2016 19:05:55 -0500 Subject: Fix broken specs. #3945 --- lib/gitlab/diff/highlight.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7dd44b6004a..caeefe5bbdd 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -27,11 +27,11 @@ module Gitlab def self.formatter @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) end def initialize(diff_file) -- cgit v1.2.1 From fed10766e533fcef2a6840deeb8d7ea1747f0c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Sat, 9 Jan 2016 01:55:31 -0500 Subject: Fix broken spec for submodule commit. #3945 --- lib/gitlab/diff/highlight.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index caeefe5bbdd..3c44abff3fb 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -85,10 +85,14 @@ module Gitlab case diff_line.type when 'new', nil - diff_line.text = new_lines[diff_line.new_pos - 1].try(:gsub!, /\A\s/, line_prefix) + line = new_lines[diff_line.new_pos - 1] when 'old' - diff_line.text = old_lines[diff_line.old_pos - 1].try(:gsub!, /\A\s/, line_prefix) + line = old_lines[diff_line.old_pos - 1] end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + diff_line.text = line.gsub!(/\A\s/, line_prefix) if line end @lines @@ -121,6 +125,10 @@ module Gitlab lines.map! { |line| " #{line}" } end end + + def submodules + @submodules ||= diff_repository.raw_repository.submodules(diff_new_ref).keys + end end end end -- cgit v1.2.1 From 96bbc145f31ad029e080ad8903445d81d6c31968 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 10:20:45 +0100 Subject: Change commit builds URL in builds API --- lib/api/builds.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 05d1b8d92ea..8c21754596b 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -28,8 +28,8 @@ module API # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; # if none provided showing all builds) # Example Request: - # GET /projects/:id/builds/commit/:sha - get ':id/builds/commit/:sha' do + # GET /projects/:id/repository/commits/:sha/builds + get ':id/repository/commits/:sha/builds' do commit = user_project.ci_commits.find_by_sha(params[:sha]) return not_found! unless commit -- cgit v1.2.1 From 4e70f2519bba83d5f9d6fd0bed80e9837e8b5fc5 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 11:15:04 +0100 Subject: Update ./doc/api/builds.md --- lib/api/entities.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cb00b392db9..a1a886d6fea 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -375,8 +375,9 @@ module API end class Build < Grape::Entity - expose :id, :status, :stage, :name, :ref, :tag, :coverage, :user + expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at + expose :user, with: UserFull expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url -- cgit v1.2.1 From 67aa0b8c4cbf762211ad178efb537f1649d91776 Mon Sep 17 00:00:00 2001 From: Drew Blessing Date: Thu, 31 Dec 2015 13:22:51 -0600 Subject: Optimize LDAP and add a search timeout --- lib/gitlab/ldap/access.rb | 8 ++++++-- lib/gitlab/ldap/adapter.rb | 24 +++++++++++++++--------- lib/gitlab/ldap/config.rb | 4 ++++ 3 files changed, 25 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..b2bdbc10d7f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,7 +32,7 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap -- cgit v1.2.1 From fc4108b38b5d0a7fc755c1977663f516b5dea0bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:27:20 +0100 Subject: Modify CI features in projects API --- lib/api/entities.rb | 4 +--- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 41 ++++++++++++++--------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8154e0f383..e3bc3316ce5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,9 +71,7 @@ module API expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } - - expose :build_allow_git_fetch, :build_timeout, :build_coverage_regex - expose :runners_token + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } end class ProjectMember < UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 563c12e4f74..a4df810e755 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -157,7 +157,7 @@ module API def attributes_for_keys(keys, custom_params = nil) attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and (params[key].empty? or params[key] == false)) + if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 31b081266a8..28c3583f351 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,9 +99,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects post do @@ -117,14 +115,12 @@ module API :namespace_id, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -149,9 +145,6 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -168,14 +161,12 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :import_url] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -194,8 +185,9 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project - end + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) + end end # Update an existing project @@ -213,9 +205,6 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project - # build_allow_git_fetch (optional) - # build_timeout (optional) - # build_coverage_regex (optional) # Example Request # PUT /projects/:id put ':id' do @@ -230,10 +219,7 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level, - :build_allow_git_fetch, - :build_timeout, - :build_coverage_regex] + :visibility_level] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -247,7 +233,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end -- cgit v1.2.1 From 5dcfd7d06754fecb245475cc005c112677693801 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 11 Jan 2016 15:49:25 +0100 Subject: Add TODO notice to build trace feature in CI API --- lib/api/builds.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 8c21754596b..d3f4e33ebbf 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -61,6 +61,10 @@ module API # build_id (required) - The ID of a build # Example Request: # GET /projects/:id/build/:build_id/trace + # + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. get ':id/builds/:build_id/trace' do build = get_build(params[:build_id]) return not_found!(build) unless build -- cgit v1.2.1 From 35b501f30ae9e121151ad6a2140d036e5ef3b0f5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 11 Jan 2016 16:51:01 +0100 Subject: Tag all transaction metrics with an "action" tag Without this it's impossible to find out what methods/views/queries are executed by a certain controller or Sidekiq worker. While this will increase the total number of series it should stay within reasonable limits due to the amount of "actions" being small enough. --- lib/gitlab/metrics/rack_middleware.rb | 6 ++---- lib/gitlab/metrics/sidekiq_middleware.rb | 7 +------ lib/gitlab/metrics/transaction.rb | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e7a2f26d48b..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -39,10 +39,8 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 73131cc6ef2..86606b1c6d6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -6,11 +6,15 @@ module Gitlab attr_reader :tags, :values + attr_accessor :action + def self.current Thread.current[THREAD_KEY] end - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] @started_at = nil @@ -18,6 +22,7 @@ module Gitlab @values = Hash.new(0) @tags = {} + @action = action end def duration @@ -70,7 +75,15 @@ module Gitlab end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) end def sidekiq? -- cgit v1.2.1 From 41bcab118ae8be1baee0d750d99f46f584bd2fd4 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:05:43 +0800 Subject: Fix #9963 reference_filter "Encoding::CompatibilityError" bug with some complex URL; --- lib/banzai/filter/reference_filter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 7198a8b03e2..5dd6d2fe3c7 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,7 +133,7 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,7 +170,7 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 932a247f5fb4a14b3e096ec88a73f8fce481eebb Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 12 Jan 2016 17:32:25 +0800 Subject: Use CGI.escape instead of URI.escape, because URI is obsoleted. ref: https://github.com/ruby/ruby/commit/238b979f1789f95262a267d8df6239806f2859cc --- lib/gitlab/gitlab_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..59926084d07 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source, '/') #Issues && Comments issues = client.issues(project_identifier) -- cgit v1.2.1 From 5679ee0120ab45829b847d55d3a2253735856b3f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 14:59:30 +0100 Subject: Track memory allocated during a transaction This gives a very rough estimate of how much memory is allocated during a transaction. This only works reliably when using a single-threaded application server and a Ruby implementation with a GIL as otherwise memory allocated by other threads might skew the statistics. Sadly there's no way around this as Ruby doesn't provide a reliable way of gathering accurate object sizes upon allocation on a per-thread basis. --- lib/gitlab/metrics/transaction.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 86606b1c6d6..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -23,20 +23,29 @@ module Gitlab @values = Hash.new(0) @tags = {} @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end @@ -65,7 +74,7 @@ module Gitlab end def track_self - values = { duration: duration } + values = { duration: duration, allocated_memory: allocated_memory } @values.each do |name, value| values[name] = value -- cgit v1.2.1 From ac6a10f3e88c5d2081b8638df63016089517a844 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Tue, 12 Jan 2016 12:29:10 -0200 Subject: Codestyle changes --- lib/api/users.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 01fd90139b0..fd2128bd179 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -303,10 +303,10 @@ module API if !user not_found!('User') - elsif !user.ldap_blocked? - user.activate - else + elsif user.ldap_blocked? forbidden!('LDAP blocked users cannot be unblocked by the API') + else + user.activate end end end -- cgit v1.2.1 From 355c341fe7c6b7c503386cf0b0e39cc02dbc8902 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 12 Jan 2016 15:41:22 +0100 Subject: Stop tracking call stacks for instrumented views Where a vew is called from doesn't matter as much. We already know what action they belong to and this is more than enough information. By removing the file/line number from the list of tags we should also be able to reduce the number of series stored in InfluxDB. --- lib/gitlab/metrics.rb | 14 -------------- lib/gitlab/metrics/subscribers/action_view.rb | 10 +--------- 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 44356a0e42c..cdf7c168ff2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -36,20 +36,6 @@ module Gitlab @pool end - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7c0105d543a..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -33,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction -- cgit v1.2.1 From ab2c6cc01ff26e07db15110e037e72159c48dc53 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 12 Jan 2016 18:32:18 +0100 Subject: Add some fixes --- lib/api/api.rb | 1 - lib/api/entities.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 266b5f48f8f..1a1340c428c 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,7 +54,6 @@ module API mount Keys mount Tags mount Triggers - mount Builds end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index a1a886d6fea..e19f9f8d75a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -377,7 +377,7 @@ module API class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at - expose :user, with: UserFull + expose :user, with: User expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url -- cgit v1.2.1 From f1f9b5f7d388c6d7a0938229c9211beddb2fd6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 12:53:54 -0500 Subject: Small fixes from code review. #3945 --- lib/gitlab/diff/highlight.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 3c44abff3fb..d0137ab5f08 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -125,10 +125,6 @@ module Gitlab lines.map! { |line| " #{line}" } end end - - def submodules - @submodules ||= diff_repository.raw_repository.submodules(diff_new_ref).keys - end end end end -- cgit v1.2.1 From efb3395b4fc0425ebbc2437ad03f0cd5fc851863 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 12 Jan 2016 19:32:44 +0100 Subject: Remove blank line --- lib/api/api.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index a9e1913f0f2..098dd975840 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,7 +54,6 @@ module API mount Keys mount Tags mount Triggers - mount Variables end end -- cgit v1.2.1 From 6e3358a5077f5c6052e733722cd6baa63e43c081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 17:36:08 -0500 Subject: Remove no longer required code. #3945 --- lib/gitlab/diff/highlight.rb | 51 ++++++++------------------------------------ 1 file changed, 9 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index d0137ab5f08..14cf98e24a2 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -44,74 +44,41 @@ module Gitlab return [] if @lines.empty? extract_line_prefixes - - @code = unescape_html(raw_content) - @highlighted_code = formatter.format(lexer.lex(@code)) - - is_diff_line? ? update_diff_lines : @highlighted_code.lines + update_diff_lines end private - def is_diff_line? - @lines.first.is_a?(Gitlab::Diff::Line) - end - def text_lines - @text_lines ||= (is_diff_line? ? @lines.map(&:text) : @lines) - end - - def raw_content - @raw_content ||= text_lines.join(is_diff_line? ? "\n" : nil) + @text_lines ||= @lines.map(&:text) end def extract_line_prefixes - @diff_line_prefixes ||= begin - if is_diff_line? - text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } - else - [] - end - end + @diff_line_prefixes ||= text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } end def update_diff_lines - @highlighted_code.lines.each_with_index do |line, i| - diff_line = @lines[i] + @lines.each_with_index do |line, i| line_prefix = @diff_line_prefixes[i] || ' ' # ignore highlighting for "match" lines - next if diff_line.type == 'match' + next if line.type == 'match' - case diff_line.type + case line.type when 'new', nil - line = new_lines[diff_line.new_pos - 1] + highlighted_line = new_lines[line.new_pos - 1] when 'old' - line = old_lines[diff_line.old_pos - 1] + highlighted_line = old_lines[line.old_pos - 1] end # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - diff_line.text = line.gsub!(/\A\s/, line_prefix) if line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if line end @lines end - def lexer - Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new - end - - def unescape_html(content) - text = CGI.unescapeHTML(content) - text.gsub!(' ', ' ') - text - end - - def formatter - self.class.formatter - end - def old_lines @old_lines ||= begin lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) -- cgit v1.2.1 From 7307fa48e70a9fb42902c8a84479d4cf669640aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 19:28:31 -0500 Subject: Fix broken specs. #3945 --- lib/gitlab/diff/highlight.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 14cf98e24a2..0b6a348acbc 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if highlighted_line end @lines -- cgit v1.2.1 From a43fd5ce6d369c35b3ea421f045721d20f1c8ada Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 12 Jan 2016 21:34:23 -0500 Subject: Disable colorization if STDOUT is not a tty --- lib/tasks/gitlab/task_helpers.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..8c63877e51c 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,8 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From d44653da1f74c2c15fe7ec3f8aa9b16563ffebd6 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 12:16:27 +0100 Subject: Add some fixes after review --- lib/api/entities.rb | 8 +------- lib/api/triggers.rb | 11 +++++------ 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 37c483b45ec..1108277aabf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,13 +367,7 @@ module API end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at - expose :last_used do |repo_obj, _options| - if repo_obj.respond_to?(:last_trigger_request) - request = repo_obj.last_trigger_request - request.created_at if request - end - end + expose :token, :created_at, :updated_at, :deleted_at, :last_used end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 25bb8aef20b..5e4964f446c 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -73,10 +73,10 @@ module API authenticate! authorize_admin_project - triggers = user_project.triggers.where(token: params[:token]) - return not_found!('Trigger') if triggers.empty? + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger - present triggers.first, with: Entities::Trigger + present trigger, with: Entities::Trigger end # Create trigger @@ -89,8 +89,7 @@ module API authenticate! authorize_admin_project - trigger = user_project.triggers.new - trigger.save + trigger = user_project.triggers.create present trigger, with: Entities::Trigger end @@ -106,7 +105,7 @@ module API authenticate! authorize_admin_project - trigger = user_project.triggers.where(token: params[:token]).first + trigger = user_project.triggers.find_by(token: params[:token].to_s) return not_found!('Trigger') unless trigger trigger.destroy -- cgit v1.2.1 From 23671600150cb022a09a77b8ea56a9465f19a013 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:29:48 +0100 Subject: Make the metrics sampler interval configurable --- lib/gitlab/metrics.rb | 3 ++- lib/gitlab/metrics/sampler.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cdf7c168ff2..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -13,7 +13,8 @@ module Gitlab timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port] + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 } end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 1ea425bc904..c2913841de3 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,7 +7,7 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) + def initialize(interval = Metrics.settings[:sample_interval]) @interval = interval @metrics = [] -- cgit v1.2.1 From df548285804fdc40ac7c4f36601e87a534792a4a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 12:47:11 +0100 Subject: Add some fixes after review --- lib/api/variables.rb | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index cc038e5731d..0c3fb5c8a77 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -27,11 +27,11 @@ module API # GET /projects/:id/variables/:key get ':id/variables/:key' do key = params[:key] - variables = user_project.variables.where(key: key) + variable = user_project.variables.find_by(key: key.to_s) - return not_found!('Variable') if variables.empty? + return not_found!('Variable') unless variable - present variables.first, with: Entities::Variable + present variable, with: Entities::Variable end # Create a new variable in project @@ -46,10 +46,12 @@ module API required_attributes! [:key, :value] variable = user_project.variables.create(key: params[:key], value: params[:value]) - return render_validation_error!(variable) unless variable.valid? - variable.save! - present variable, with: Entities::Variable + if variable.valid? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end end # Update existing variable of a project @@ -61,14 +63,16 @@ module API # Example Request: # PUT /projects/:id/variables/:key put ':id/variables/:key' do - variable = user_project.variables.where(key: params[:key]).first + variable = user_project.variables.find_by(key: params[:key].to_s) return not_found!('Variable') unless variable - variable.value = params[:value] - variable.save! - - present variable, with: Entities::Variable + attrs = attributes_for_keys [:value] + if variable.update(attrs) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end end # Delete existing variable of a project @@ -79,10 +83,9 @@ module API # Exanoke Reqyest: # DELETE /projects/:id/variables/:key delete ':id/variables/:key' do - variable = user_project.variables.where(key: params[:key]).first + variable = user_project.variables.find_by(key: params[:key].to_s) return not_found!('Variable') unless variable - variable.destroy present variable, with: Entities::Variable -- cgit v1.2.1 From 057eb824b5d7f38547506303dc80da6164715420 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 13 Jan 2016 12:57:46 +0100 Subject: Randomize metrics sample intervals Sampling data at a fixed interval means we can potentially miss data from events occurring between sampling intervals. For example, say we sample data every 15 seconds but Unicorn workers get killed after 10 seconds. In this particular case it's possible to miss interesting data as the sampler will never get to actually submitting data. To work around this (at least for the most part) the sampling interval is randomized as following: 1. Take the user specified sampling interval (15 seconds by default) 2. Divide it by 2 (referred to as "half" below) 3. Generate a range (using a step of 0.1) from -"half" to "half" 4. Every time the sampler goes to sleep we'll grab the user provided interval and add a randomly chosen "adjustment" to it while making sure we don't pick the same value twice in a row. For a specified timeout of 15 this means the actual intervals can be anywhere between 7.5 and 22.5, but never can the same interval be used twice in a row. The rationale behind this change is that on dev.gitlab.org I'm sometimes seeing certain Gitlab::Git/Rugged objects being retained, but only for a few minutes every 24 hours. Knowing the code of Gitlab and how much memory it uses/leaks I suspect we're missing data due to workers getting terminated before the sampler can write its data to InfluxDB. --- lib/gitlab/metrics/sampler.rb | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index c2913841de3..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -8,8 +8,13 @@ module Gitlab class Sampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - @interval = interval - @metrics = [] + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -102,6 +107,23 @@ module Gitlab def sidekiq? Sidekiq.server? end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end end end end -- cgit v1.2.1 From 97338496188add9ec8d192c7e78f6a6040befffa Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 15:17:59 +0100 Subject: Add some fixes after review --- lib/api/builds.rb | 24 +++++++++++++----------- lib/api/entities.rb | 6 ++---- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index d3f4e33ebbf..1337e1bb452 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -35,6 +35,7 @@ module API builds = commit.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) + present paginate(builds), with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end @@ -118,25 +119,26 @@ module API helpers do def get_build(id) - user_project.builds.where(id: id).first + user_project.builds.find_by(id: id.to_i) end def filter_builds(builds, scope) - available_scopes = Ci::Build.available_statuses + return builds if scope.nil? || scope.empty? + + available_statuses = Ci::Build.available_statuses scope = - if scope.is_a?(String) || scope.is_a?(Symbol) - available_scopes & [scope.to_s] - elsif scope.is_a?(Array) - available_scopes & scope - elsif scope.respond_to?(:to_h) - available_scopes & scope.to_h.values + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values else - [] + ['unknown'] end - return builds if scope.empty? + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? - builds.where(status: scope) + builds.where(status: available_statuses && scope) end def authorize_manage_builds! diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e19f9f8d75a..f0816a46529 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -378,18 +378,16 @@ module API expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :user, with: User + # TODO: download_url in Ci:Build model is an GitLab Web Interface URL, not API URL. We should think on some API + # for downloading of artifacts (see: https://gitlab.com/gitlab-org/gitlab-ce/issues/4255) expose :download_url do |repo_obj, options| if options[:user_can_download_artifacts] repo_obj.download_url - else - nil end end expose :commit, with: RepoCommit do |repo_obj, _options| if repo_obj.respond_to?(:commit) repo_obj.commit.commit_data - else - nil end end expose :runner, with: Runner -- cgit v1.2.1 From 0f0af19139db71255934e9a7a5b5cd86420b7186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 13 Jan 2016 11:39:15 -0500 Subject: Little refactor for usage of html_safe. #3945 --- lib/gitlab/diff/highlight.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0b6a348acbc..f940b57d596 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -22,7 +22,7 @@ module Gitlab content = blob.data lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new - formatter.format(lexer.lex(content)).lines + formatter.format(lexer.lex(content)).lines.map!(&:html_safe) end def self.formatter @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if highlighted_line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix).html_safe if highlighted_line end @lines -- cgit v1.2.1 From 9e701ccd48ed442124509aeb68fe6788579efdde Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 13 Jan 2016 18:47:39 +0100 Subject: Fix some typos --- lib/api/variables.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 0c3fb5c8a77..d9a055f6c92 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -18,7 +18,7 @@ module API present paginate(variables), with: Entities::Variable end - # Get specifica variable of a project + # Get specific variable of a project # # Parameters: # id (required) - The ID of a project @@ -80,7 +80,7 @@ module API # Parameters: # id (required) - The ID of a project # key (required) - The ID of a variable - # Exanoke Reqyest: + # Example Request: # DELETE /projects/:id/variables/:key delete ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key].to_s) -- cgit v1.2.1 From 0c10aee59677e2dadfef6538a74fe1e28fcdd37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 13 Jan 2016 19:42:36 +0100 Subject: Ensure the API doesn't return notes that the current user shouldn't see --- lib/api/notes.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 3efdfe2d46e..174473f5371 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -20,7 +20,19 @@ module API # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - present paginate(@noteable.notes), with: Entities::Note + + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note end # Get a single +noteable+ note @@ -35,7 +47,12 @@ module API get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) @note = @noteable.notes.find(params[:note_id]) - present @note, with: Entities::Note + + if @note.cross_reference_not_visible_for?(current_user) + not_found!("Note") + else + present @note, with: Entities::Note + end end # Create a new +noteable+ note -- cgit v1.2.1 From c179b48c97be22ef55ef9f5874984f7359fb12f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 13 Jan 2016 21:04:53 -0500 Subject: Use #sub instead of #gsub!. #3945 * This is because is not a good idea to modify the original lines * Also I run into this issue https://gitlab.com/gitlab-org/gitlab_git/issues/14 which is returning Diff Lines with the same @new_pos value. --- lib/gitlab/diff/highlight.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f940b57d596..e76a6f27856 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix).html_safe if highlighted_line + line.text = highlighted_line.sub(/\A\s/, line_prefix).html_safe if highlighted_line end @lines -- cgit v1.2.1 From dd6fc01ff8a073880b67a323a547edeb5d63f167 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 14 Jan 2016 03:31:27 -0200 Subject: fixed LDAP activation on login to use new ldap_blocked state --- lib/gitlab/ldap/access.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index ebd9260ad5d..a659d179b5f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -40,9 +40,7 @@ module Gitlab user.ldap_block false else - if (user.blocked? && !ldap_config.block_auto_created_users) || user.ldap_blocked? - user.activate - end + user.activate if user.ldap_blocked? true end else -- cgit v1.2.1 From 2a20c0e590be2f673188f1a0d27409a714800c89 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Jan 2016 15:46:12 +0100 Subject: Prevent circular reference error in Banzai --- lib/banzai/cross_project_reference.rb | 2 -- lib/banzai/filter.rb | 1 - lib/banzai/filter/abstract_reference_filter.rb | 2 -- lib/banzai/filter/autolink_filter.rb | 1 - lib/banzai/filter/commit_range_reference_filter.rb | 2 -- lib/banzai/filter/commit_reference_filter.rb | 2 -- lib/banzai/filter/emoji_filter.rb | 1 - lib/banzai/filter/external_issue_reference_filter.rb | 2 -- lib/banzai/filter/external_link_filter.rb | 1 - lib/banzai/filter/issue_reference_filter.rb | 2 -- lib/banzai/filter/label_reference_filter.rb | 2 -- lib/banzai/filter/markdown_filter.rb | 1 - lib/banzai/filter/merge_request_reference_filter.rb | 2 -- lib/banzai/filter/redactor_filter.rb | 1 - lib/banzai/filter/reference_filter.rb | 1 - lib/banzai/filter/reference_gatherer_filter.rb | 1 - lib/banzai/filter/relative_link_filter.rb | 1 - lib/banzai/filter/sanitization_filter.rb | 1 - lib/banzai/filter/snippet_reference_filter.rb | 2 -- lib/banzai/filter/syntax_highlight_filter.rb | 1 - lib/banzai/filter/table_of_contents_filter.rb | 1 - lib/banzai/filter/task_list_filter.rb | 1 - lib/banzai/filter/upload_link_filter.rb | 1 - lib/banzai/filter/user_reference_filter.rb | 2 -- lib/banzai/lazy_reference.rb | 2 -- lib/banzai/pipeline.rb | 2 -- lib/banzai/pipeline/asciidoc_pipeline.rb | 2 -- lib/banzai/pipeline/atom_pipeline.rb | 2 -- lib/banzai/pipeline/base_pipeline.rb | 1 - lib/banzai/pipeline/combined_pipeline.rb | 2 -- lib/banzai/pipeline/description_pipeline.rb | 2 -- lib/banzai/pipeline/email_pipeline.rb | 2 -- lib/banzai/pipeline/full_pipeline.rb | 2 -- lib/banzai/pipeline/gfm_pipeline.rb | 2 -- lib/banzai/pipeline/note_pipeline.rb | 2 -- lib/banzai/pipeline/plain_markdown_pipeline.rb | 2 -- lib/banzai/pipeline/post_process_pipeline.rb | 2 -- lib/banzai/pipeline/reference_extraction_pipeline.rb | 2 -- lib/banzai/pipeline/single_line_pipeline.rb | 2 -- lib/banzai/reference_extractor.rb | 2 -- lib/gitlab/markdown/pipeline.rb | 2 -- lib/gitlab/reference_extractor.rb | 2 -- 42 files changed, 69 deletions(-) (limited to 'lib') diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index ba2866e1efa..0257848b6bc 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Common methods for ReferenceFilters that support an optional cross-project # reference. diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index fd4fe024252..905c4c0144e 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' module Banzai module Filter diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index b2db10e6864..cdbaecf8d90 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # Issues, Merge Requests, Snippets, Commits and Commit Ranges share diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index da4ee80c1b5..856f56fb175 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index e67cd45ab9b..470727ee312 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit range references with links. diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 9e57608b483..713a56ba949 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit references with links. diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 86838e1483c..5952a031626 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,5 +1,4 @@ require 'action_controller' -require 'banzai' require 'gitlab_emoji' require 'html/pipeline/filter' diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 6136e73c096..edc26386903 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces external issue tracker references with links. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index ac87b9820af..8d368f3b9e7 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 51180cb901a..9f08aa36e8b 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces issue references with links. References to diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a3a7a23c1e6..95e7d209119 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces label references with links. diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index d09cf41df39..0659fed1419 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 755b946a34b..57c71708992 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces merge request references with links. References diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 66f77902319..7141ed7c9bd 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 5dd6d2fe3c7..20bd4f7ee6e 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index bef04112919..86d484feb90 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 66f166939e4..41380627d39 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d03e3ae4b3c..3f49d492f2f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 1ad5df96f85..c870a42f741 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces snippet references with links. References to diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index c889cc1e97c..8c5855e5ffc 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 9b3e67206d5..4056dcd6d64 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index d0ce13003a5..66608c9859c 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'task_list/filter' module Banzai diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 1a1d0aad8ca..f642aee0967 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 964ab60f614..24f16f8b547 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces user or group references with links. diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb index 073ec5d9801..1095b4debc7 100644 --- a/lib/banzai/lazy_reference.rb +++ b/lib/banzai/lazy_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai class LazyReference def self.load(refs) diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb index 4e017809d9d..142a9962eb1 100644 --- a/lib/banzai/pipeline.rb +++ b/lib/banzai/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline def self.[](name) diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb index 5e76a817be5..f1331c0ebf9 100644 --- a/lib/banzai/pipeline/asciidoc_pipeline.rb +++ b/lib/banzai/pipeline/asciidoc_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AsciidocPipeline < BasePipeline diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 957f352aec5..9694e4bc23f 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AtomPipeline < FullPipeline diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index cd30009e5c0..db5177db7b3 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline' module Banzai diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb index f3bf1809d18..9485199132e 100644 --- a/lib/banzai/pipeline/combined_pipeline.rb +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline module CombinedPipeline diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 94c2cb165a5..20e24ace352 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class DescriptionPipeline < FullPipeline diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 14356145a35..e47c384afc1 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class EmailPipeline < FullPipeline diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index 72395a5d50e..d47ddfda4be 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 838155e8831..b7a38ea8427 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class GfmPipeline < BasePipeline diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb index 89335143852..7890f20f716 100644 --- a/lib/banzai/pipeline/note_pipeline.rb +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class NotePipeline < FullPipeline diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 998fd75daa2..3fbc681457b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 148f24b6ce1..bd338c045f3 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PostProcessPipeline < BasePipeline diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb index 4f9bc9fcccc..eaddccd30a5 100644 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class ReferenceExtractionPipeline < BasePipeline diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index a3c9d4f43aa..8b84ab401df 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class SingleLinePipeline < GfmPipeline diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 2c197d31898..f4079538ec5 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 8f3f43c0e91..699d8b9fc07 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab module Markdown class Pipeline diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 4164e998dd1..4d830aa45e1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor -- cgit v1.2.1 From 3183092ca94b14d6e61f5e8ba51069554646baf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 14 Jan 2016 12:08:44 +0100 Subject: Add pagination headers to already paginated API resources --- lib/api/helpers.rb | 30 +++++++++++++++++++----------- lib/api/notes.rb | 12 +++--------- 2 files changed, 22 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..312ef90915f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,11 +97,9 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) - add_pagination_headers(paginated, per_page) - - paginated + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end end def authenticate! @@ -327,16 +325,26 @@ module API private - def add_pagination_headers(paginated, per_page) + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) request_url = request.url.split('?').first links = [] - links << %(<#{request_url}?page=#{paginated.current_page - 1}&per_page=#{per_page}>; rel="prev") unless paginated.first_page? - links << %(<#{request_url}?page=#{paginated.current_page + 1}&per_page=#{per_page}>; rel="next") unless paginated.last_page? - links << %(<#{request_url}?page=1&per_page=#{per_page}>; rel="first") - links << %(<#{request_url}?page=#{paginated.total_pages}&per_page=#{per_page}>; rel="last") + links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page? + links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page? + links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first") + links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last") - header 'Link', links.join(', ') + links.join(', ') end def abilities diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..ebd9e97148c 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -22,17 +22,11 @@ module API @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. + # by the current user. notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). + @noteable.notes. reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note + present paginate(Kaminari.paginate_array(notes)), with: Entities::Note end # Get a single +noteable+ note -- cgit v1.2.1 From f5d530865875440d69217cf249715bffaa3d11b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 11:59:10 +0100 Subject: Add implementation of StringPath class `StringPath` class is something similar to Ruby's `Pathname` class, but does not involve any IO operations. `StringPath` objects require passing string representation of path, and array of paths that represents universe to constructor to be intantiated. --- lib/gitlab/string_path.rb | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/gitlab/string_path.rb (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb new file mode 100644 index 00000000000..be65b41dff5 --- /dev/null +++ b/lib/gitlab/string_path.rb @@ -0,0 +1,35 @@ +module Gitlab + ## + # Class that represents a path to a file or directory + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # + class StringPath + def initialize(path, universe) + @path = path + @universe = universe + end + + def absolute? + @path.start_with?('/') + end + + def relative? + !absolute? + end + + def directory? + @path.end_with?('/') + end + + def file? + !directory? + end + + def to_s + @path + end + end +end -- cgit v1.2.1 From 73d2c7a553ca239cdce04af793992fd579ad3e4b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 12:32:21 +0100 Subject: Add new methods to StringPath --- lib/gitlab/string_path.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index be65b41dff5..9ccf54bd62f 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -7,11 +7,17 @@ module Gitlab # # class StringPath + attr_reader :path, :universe + def initialize(path, universe) @path = path @universe = universe end + def to_s + @path + end + def absolute? @path.start_with?('/') end @@ -28,8 +34,17 @@ module Gitlab !directory? end - def to_s - @path + def files + raise NotImplementedError + end + + def basename + name = @path.split(::File::SEPARATOR).last + directory? ? name + ::File::SEPARATOR : name + end + + def ==(other) + @path == other.path && @universe == other.universe end end end -- cgit v1.2.1 From 80a71576ba27d84b3406a8b929328359e2edc9da Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Dec 2015 12:55:50 +0100 Subject: Use `Gitlab::StringPath` in CI build artifacts controller --- lib/gitlab/string_path.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 9ccf54bd62f..3aa6200b572 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -34,6 +34,18 @@ module Gitlab !directory? end + def has_parent? + raise NotImplementedError + end + + def parent + raise NotImplementedError + end + + def directories + raise NotImplementedError + end + def files raise NotImplementedError end -- cgit v1.2.1 From 518b206287318006f9b57382a747b1474b6795a4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 19 Dec 2015 09:31:52 +0100 Subject: Add `parent` iteration implementation to `StringPath` --- lib/gitlab/string_path.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 3aa6200b572..3add56d2213 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -35,11 +35,12 @@ module Gitlab end def has_parent? - raise NotImplementedError + @universe.include?(@path.sub(basename, '')) end def parent - raise NotImplementedError + return nil unless has_parent? + new(@path.sub(basename, '')) end def directories @@ -58,5 +59,11 @@ module Gitlab def ==(other) @path == other.path && @universe == other.universe end + + private + + def new(path) + self.class.new(path, @universe) + end end end -- cgit v1.2.1 From d382335dcd9285c9355ed04dc12c5314bca3c024 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 21 Dec 2015 10:11:15 +0100 Subject: Add implementation of remaining methods in `StringPath` --- lib/gitlab/string_path.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 3add56d2213..a2dc63db2f2 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -43,12 +43,24 @@ module Gitlab new(@path.sub(basename, '')) end + def descendants + return [] unless directory? + children = @universe.select { |entry| entry =~ /^#{@path}.+/ } + children.map { |path| new(path) } + end + + def children + descendants.select { |descendant| descendant.parent == self } + end + def directories - raise NotImplementedError + return [] unless directory? + children.select { |child| child.directory? } end def files - raise NotImplementedError + return [] unless directory? + children.select { |child| child.file? } end def basename -- cgit v1.2.1 From 37b2c5dd5521f25a7195e82538a0ffc528c3ec6d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 21 Dec 2015 12:08:04 +0100 Subject: Add support for root path for `StringPath` --- lib/gitlab/string_path.rb | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index a2dc63db2f2..d165829132a 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -1,6 +1,6 @@ module Gitlab ## - # Class that represents a path to a file or directory + # Class that represents a simplified path to a file or directory # # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. @@ -10,8 +10,9 @@ module Gitlab attr_reader :path, :universe def initialize(path, universe) - @path = path - @universe = universe + @path = prepare(path) + @universe = universe.map { |entry| prepare(entry) } + @universe.unshift('./') unless @universe.include?('./') end def to_s @@ -43,6 +44,15 @@ module Gitlab new(@path.sub(basename, '')) end + def basename + name = @path.split(::File::SEPARATOR).last + directory? ? name + ::File::SEPARATOR : name + end + + def has_descendants? + descendants.any? + end + def descendants return [] unless directory? children = @universe.select { |entry| entry =~ /^#{@path}.+/ } @@ -63,11 +73,6 @@ module Gitlab children.select { |child| child.file? } end - def basename - name = @path.split(::File::SEPARATOR).last - directory? ? name + ::File::SEPARATOR : name - end - def ==(other) @path == other.path && @universe == other.universe end @@ -77,5 +82,10 @@ module Gitlab def new(path) self.class.new(path, @universe) end + + def prepare(path) + return path if path =~ %r{^(/|\.|\.\.)} + path.dup.prepend('./') + end end end -- cgit v1.2.1 From b19e958d86f5363057f006c8dbf9a8e8762618b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 Dec 2015 11:05:22 +0100 Subject: Add support for parent directories in `StringPath` This support is not completed though, as parent directory that is first in collection returned by `directories!` is not iterable yet. --- lib/gitlab/string_path.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index d165829132a..493fceb256d 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,6 +5,7 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # + # TODO: better support for './' and '../' # class StringPath attr_reader :path, :universe @@ -45,10 +46,13 @@ module Gitlab end def basename - name = @path.split(::File::SEPARATOR).last directory? ? name + ::File::SEPARATOR : name end + def name + @path.split(::File::SEPARATOR).last + end + def has_descendants? descendants.any? end @@ -68,6 +72,10 @@ module Gitlab children.select { |child| child.directory? } end + def directories! + has_parent? ? directories.prepend(new(@path + '../')) : directories + end + def files return [] unless directory? children.select { |child| child.file? } -- cgit v1.2.1 From 1cc26e0f0e2ef62fecb688e45bc51c66c8fdf0a7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 Dec 2015 13:02:00 +0100 Subject: Improve performance of `StringPath` --- lib/gitlab/string_path.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 493fceb256d..1eb8162f805 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,15 +5,15 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO: better support for './' and '../' + # TODO: better support for '../' and './' # class StringPath attr_reader :path, :universe def initialize(path, universe) @path = prepare(path) - @universe = universe.map { |entry| prepare(entry) } - @universe.unshift('./') unless @universe.include?('./') + @universe = Set.new(universe.map { |entry| prepare(entry) }) + @universe.add('./') end def to_s @@ -64,7 +64,10 @@ module Gitlab end def children - descendants.select { |descendant| descendant.parent == self } + return [] unless directory? + return @children if @children + children = @universe.select { |entry| entry =~ %r{^#{@path}[^/]+/?$} } + @children = children.map { |path| new(path) } end def directories @@ -85,6 +88,10 @@ module Gitlab @path == other.path && @universe == other.universe end + def inspect + "#{self.class.name}: #{@path}" + end + private def new(path) -- cgit v1.2.1 From c177784d5af6b47ae613f922e075a38fc56ad711 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 29 Dec 2015 11:40:57 +0100 Subject: Use short method call in StringPath instead block --- lib/gitlab/string_path.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 1eb8162f805..af80a502bf6 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -72,7 +72,7 @@ module Gitlab def directories return [] unless directory? - children.select { |child| child.directory? } + children.select(&:directory?) end def directories! @@ -81,7 +81,7 @@ module Gitlab def files return [] unless directory? - children.select { |child| child.file? } + children.select(&:file?) end def ==(other) -- cgit v1.2.1 From 447f56036e837fc9a9c2bcaf382d38dc513a9733 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Dec 2015 09:25:59 +0100 Subject: Use metadata stored in artifacats metadata file --- lib/gitlab/string_path.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index af80a502bf6..bfeb0f852f0 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,7 +5,7 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO: better support for '../' and './' + # TODO, better support for '../' and './' # class StringPath attr_reader :path, :universe -- cgit v1.2.1 From 3de8a4620a70c886c815576dc0a30a745cbb8ccb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Dec 2015 12:21:56 +0100 Subject: Parse artifacts metadata stored in JSON format --- lib/gitlab/string_path.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index bfeb0f852f0..ad68a8ff2d6 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -10,10 +10,11 @@ module Gitlab class StringPath attr_reader :path, :universe - def initialize(path, universe) + def initialize(path, universe, metadata = []) @path = prepare(path) - @universe = Set.new(universe.map { |entry| prepare(entry) }) - @universe.add('./') + @universe = universe.map { |entry| prepare(entry) } + @universe << './' unless @universe.include?('./') + @metadata = metadata end def to_s @@ -84,6 +85,11 @@ module Gitlab children.select(&:file?) end + def metadata + index = @universe.index(@path) + @metadata[index] + end + def ==(other) @path == other.path && @universe == other.universe end -- cgit v1.2.1 From a3191463b60c8ded25a2898d5e5520ae4aff1114 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 2 Jan 2016 09:43:10 +0100 Subject: Add path sanitization to `StringPath` [ci skip] --- lib/gitlab/string_path.rb | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index ad68a8ff2d6..8310564646e 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -5,15 +5,12 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # - # TODO, better support for '../' and './' - # class StringPath attr_reader :path, :universe def initialize(path, universe, metadata = []) - @path = prepare(path) - @universe = universe.map { |entry| prepare(entry) } - @universe << './' unless @universe.include?('./') + @path = sanitize(path) + @universe = universe.map { |entry| sanitize(entry) } @metadata = metadata end @@ -60,15 +57,16 @@ module Gitlab def descendants return [] unless directory? - children = @universe.select { |entry| entry =~ /^#{@path}.+/ } - children.map { |path| new(path) } + select { |entry| entry =~ /^#{@path}.+/ } end def children return [] unless directory? return @children if @children - children = @universe.select { |entry| entry =~ %r{^#{@path}[^/]+/?$} } - @children = children.map { |path| new(path) } + + @children = select do |entry| + self.class.child?(@path, entry) + end end def directories @@ -104,9 +102,26 @@ module Gitlab self.class.new(path, @universe) end - def prepare(path) - return path if path =~ %r{^(/|\.|\.\.)} - path.dup.prepend('./') + def select + selected = @universe.select { |entry| yield entry } + selected.map { |path| new(path) } + end + + def sanitize(path) + self.class.sanitize(path) + end + + def self.sanitize(path) + # It looks like Pathname#new doesn't touch a file system, + # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe + + clean = Pathname.new(path).cleanpath.to_s + raise ArgumentError, 'Invalid path' if clean.start_with?('../') + clean + (path.end_with?('/') ? '/' : '') + end + + def self.child?(path, entry) + entry =~ %r{^#{path}[^/\s]+/?$} end end end -- cgit v1.2.1 From df41148662142ce20a77b092665f48dd4dfa7bfb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 2 Jan 2016 20:09:21 +0100 Subject: Improve path sanitization in `StringPath` --- lib/gitlab/string_path.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 8310564646e..e7d99a35869 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -57,7 +57,7 @@ module Gitlab def descendants return [] unless directory? - select { |entry| entry =~ /^#{@path}.+/ } + select { |entry| entry =~ /^#{Regexp.escape(@path)}.+/ } end def children @@ -65,7 +65,7 @@ module Gitlab return @children if @children @children = select do |entry| - self.class.child?(@path, entry) + entry =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?$} end end @@ -75,7 +75,7 @@ module Gitlab end def directories! - has_parent? ? directories.prepend(new(@path + '../')) : directories + has_parent? ? directories.prepend(parent) : directories end def files @@ -115,13 +115,12 @@ module Gitlab # It looks like Pathname#new doesn't touch a file system, # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe - clean = Pathname.new(path).cleanpath.to_s - raise ArgumentError, 'Invalid path' if clean.start_with?('../') - clean + (path.end_with?('/') ? '/' : '') - end + clean_path = Pathname.new(path).cleanpath.to_s + raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') - def self.child?(path, entry) - entry =~ %r{^#{path}[^/\s]+/?$} + prefix = './' unless clean_path =~ %r{^[\.|/]} + suffix = '/' if path.end_with?('/') || clean_path =~ /^[\.|\.\.]$/ + prefix.to_s + clean_path + suffix.to_s end end end -- cgit v1.2.1 From a7f99b67a0bf1160f41ebf4dc92c618eb13a7a10 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 13:08:49 +0100 Subject: Extract artifacts metadata implementation to separate class --- lib/gitlab/ci/build/artifacts/metadata.rb | 57 +++++++++++++++++++++++++++++++ lib/gitlab/string_path.rb | 4 +-- 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..5313182d55f --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,57 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + def initialize(file, path) + @file = file + + @path = path.sub(/^\.\//, '') + @path << '/' unless path.end_with?('/') + end + + def exists? + File.exists?(@file) + end + + def match! + raise StandardError, 'Metadata file not found !' unless exists? + paths, metadata = [], [] + + each do |line| + next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?\s} + + path, meta = line.split(' ') + paths.push(path) + metadata.push(meta) + end + + [paths, metadata.map { |meta| JSON.parse(meta) }] + end + + def to_string_path + universe, metadata = match! + ::Gitlab::StringPath.new(@path, universe, metadata) + end + + private + + def each + open do |file| + gzip = Zlib::GzipReader.new(file) + gzip.each_line { |line| yield line } + gzip.close + end + end + + def open + File.open(@file) { |file| yield file } + end + end + end + end + end +end diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index e7d99a35869..9948502e8ea 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -75,7 +75,7 @@ module Gitlab end def directories! - has_parent? ? directories.prepend(parent) : directories + @path =~ %r{^\./[^/]/} ? directories.prepend(parent) : directories end def files @@ -119,7 +119,7 @@ module Gitlab raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') prefix = './' unless clean_path =~ %r{^[\.|/]} - suffix = '/' if path.end_with?('/') || clean_path =~ /^[\.|\.\.]$/ + suffix = '/' if path.end_with?('/') || ['.', '..'].include?(clean_path) prefix.to_s + clean_path + suffix.to_s end end -- cgit v1.2.1 From f948c00757ca9529817c7368610b0c0d6734d48f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 13:40:42 +0100 Subject: Do not depend on universe when checking parent in `StringPath` --- lib/gitlab/string_path.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 9948502e8ea..4d024b3ff73 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -7,6 +7,7 @@ module Gitlab # class StringPath attr_reader :path, :universe + attr_accessor :name def initialize(path, universe, metadata = []) @path = sanitize(path) @@ -35,7 +36,7 @@ module Gitlab end def has_parent? - @universe.include?(@path.sub(basename, '')) + nodes > 1 end def parent @@ -48,7 +49,7 @@ module Gitlab end def name - @path.split(::File::SEPARATOR).last + @name || @path.split(::File::SEPARATOR).last end def has_descendants? @@ -75,7 +76,11 @@ module Gitlab end def directories! - @path =~ %r{^\./[^/]/} ? directories.prepend(parent) : directories + return directories unless has_parent? && directory? + + dotted_parent = parent + dotted_parent.name = '..' + directories.prepend(dotted_parent) end def files @@ -88,6 +93,10 @@ module Gitlab @metadata[index] end + def nodes + @path.count('/') + (file? ? 1 : 0) + end + def ==(other) @path == other.path && @universe == other.universe end -- cgit v1.2.1 From a5e1905d28e490fb4734bff0e02a1ecff4c7c029 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 14:00:49 +0100 Subject: Render 404 when artifacts path is invalid --- lib/gitlab/ci/build/artifacts/metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 5313182d55f..d5c3cd10e48 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -22,7 +22,7 @@ module Gitlab paths, metadata = [], [] each do |line| - next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?\s} + next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} path, meta = line.split(' ') paths.push(path) -- cgit v1.2.1 From cd3b8bbd2f8e7ad75a453441f83c46aeb1d37353 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 14:18:06 +0100 Subject: Add method that checks if path exists in `StringPath` --- lib/gitlab/string_path.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index 4d024b3ff73..a6234d34e7d 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -19,6 +19,10 @@ module Gitlab @path end + def exists? + @path == './' || @universe.include?(@path) + end + def absolute? @path.start_with?('/') end -- cgit v1.2.1 From 1b1793c2530d7003d8baa5aa1912a4ab258d4a1c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jan 2016 15:07:49 +0100 Subject: Show file size in artifacts browser using metadata --- lib/gitlab/ci/build/artifacts/metadata.rb | 5 ++--- lib/gitlab/string_path.rb | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d5c3cd10e48..1f3000e7c8a 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -23,13 +23,12 @@ module Gitlab each do |line| next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} - path, meta = line.split(' ') paths.push(path) metadata.push(meta) - end + end - [paths, metadata.map { |meta| JSON.parse(meta) }] + [paths, metadata.map { |meta| JSON.parse(meta, symbolize_names: true) }] end def to_string_path diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb index a6234d34e7d..774d4244a2a 100644 --- a/lib/gitlab/string_path.rb +++ b/lib/gitlab/string_path.rb @@ -19,10 +19,6 @@ module Gitlab @path end - def exists? - @path == './' || @universe.include?(@path) - end - def absolute? @path.start_with?('/') end @@ -94,13 +90,17 @@ module Gitlab def metadata index = @universe.index(@path) - @metadata[index] + @metadata[index] || {} end def nodes @path.count('/') + (file? ? 1 : 0) end + def exists? + @path == './' || @universe.include?(@path) + end + def ==(other) @path == other.path && @universe == other.universe end @@ -112,7 +112,7 @@ module Gitlab private def new(path) - self.class.new(path, @universe) + self.class.new(path, @universe, @metadata) end def select -- cgit v1.2.1 From 387b27813d1d496c015f4f174812b4761c32648d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 8 Jan 2016 12:35:49 +0100 Subject: Change format of artifacts metadata from text to binary 0.0.1 This changes the format of metadata to handle paths, that may contain whitespace characters, new line characters and non-UTF-8 characters. Now those paths along with metadata in JSON format are stored as length-prefixed strings (uint32 prefix). Metadata file has a custom format: 1. First string field is metadata version field (string) 2. Second string field is metadata errors field (JSON strong) 3. All subsequent fields is pair of path (string) and path metadata in JSON format. Path's metadata contains all fields that where possible to extract from ZIP archive like date of modification, CRC, compressed size, uncompressed size and comment. --- lib/gitlab/ci/build/artifacts/metadata.rb | 68 ++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1f3000e7c8a..d90a64fdbb8 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -17,18 +17,33 @@ module Gitlab File.exists?(@file) end + def full_version + gzip do|gz| + read_string(gz) do |size| + raise StandardError, 'Artifacts metadata file empty!' unless size + end + end + end + + def version + full_version.match(/\w+ (\d+\.\d+\.\d+)/).captures.first + end + + def errors + gzip do|gz| + read_string(gz) # version + JSON.parse(read_string(gz)) + end + end + def match! raise StandardError, 'Metadata file not found !' unless exists? - paths, metadata = [], [] - each do |line| - next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s} - path, meta = line.split(' ') - paths.push(path) - metadata.push(meta) + gzip do |gz| + read_string(gz) # version field + read_string(gz) # errors field + iterate_entries(gz) end - - [paths, metadata.map { |meta| JSON.parse(meta, symbolize_names: true) }] end def to_string_path @@ -38,11 +53,44 @@ module Gitlab private - def each + def iterate_entries(gz) + paths, metadata = [], [] + + until gz.eof? do + begin + path = read_string(gz) + meta = read_string(gz) + + next unless path =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + + paths.push(path) + metadata.push(JSON.parse(meta, symbolize_names: true)) + rescue JSON::ParserError + next + end + end + + [paths, metadata] + end + + def read_string_size(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_string_size(gz) + yield string_size if block_given? + return false unless string_size + gz.read(string_size).chomp + end + + def gzip open do |file| gzip = Zlib::GzipReader.new(file) - gzip.each_line { |line| yield line } + result = yield gzip gzip.close + result end end -- cgit v1.2.1 From 61fb47a43202332fe9ac57847996da929ba42d3f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 9 Jan 2016 14:41:43 +0100 Subject: Simplify implementation of build artifacts browser (refactoring) --- lib/gitlab/ci/build/artifacts/metadata.rb | 74 ++++++------- lib/gitlab/ci/build/artifacts/metadata/path.rb | 114 ++++++++++++++++++++ lib/gitlab/string_path.rb | 139 ------------------------- 3 files changed, 153 insertions(+), 174 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata/path.rb delete mode 100644 lib/gitlab/string_path.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d90a64fdbb8..996b5d91ff2 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,65 +6,54 @@ module Gitlab module Build module Artifacts class Metadata - def initialize(file, path) - @file = file - - @path = path.sub(/^\.\//, '') - @path << '/' unless path.end_with?('/') - end + VERSION_PATTERN = '[\w\s]+(\d+\.\d+\.\d+)' + attr_reader :file, :path, :full_version - def exists? - File.exists?(@file) - end - - def full_version - gzip do|gz| - read_string(gz) do |size| - raise StandardError, 'Artifacts metadata file empty!' unless size - end - end + def initialize(file, path) + @file, @path = file, path + @full_version = read_version + @path << '/' unless path.end_with?('/') || path.empty? end def version - full_version.match(/\w+ (\d+\.\d+\.\d+)/).captures.first + @full_version.match(/#{VERSION_PATTERN}/).captures.first end def errors gzip do|gz| read_string(gz) # version - JSON.parse(read_string(gz)) + errors = read_string(gz) + raise StandardError, 'Errors field not found!' unless errors + JSON.parse(errors) end end def match! - raise StandardError, 'Metadata file not found !' unless exists? - gzip do |gz| - read_string(gz) # version field - read_string(gz) # errors field - iterate_entries(gz) + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) end end - def to_string_path - universe, metadata = match! - ::Gitlab::StringPath.new(@path, universe, metadata) + def to_path + Path.new(@path, *match!) end private - def iterate_entries(gz) + def match_entries(gz) paths, metadata = [], [] - + child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + until gz.eof? do begin path = read_string(gz) meta = read_string(gz) - next unless path =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?$} - + next unless path =~ child_pattern + paths.push(path) - metadata.push(JSON.parse(meta, symbolize_names: true)) + metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) rescue JSON::ParserError next end @@ -73,16 +62,31 @@ module Gitlab [paths, metadata] end - def read_string_size(gz) + def read_version + gzip do|gz| + version_string = read_string(gz) + + unless version_string + raise StandardError, 'Artifacts metadata file empty!' + end + + unless version_string =~ /^#{VERSION_PATTERN}/ + raise StandardError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) binary = gz.read(4) binary.unpack('L>')[0] if binary end def read_string(gz) - string_size = read_string_size(gz) - yield string_size if block_given? + string_size = read_uint32(gz) return false unless string_size - gz.read(string_size).chomp + gz.read(string_size) end def gzip diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb new file mode 100644 index 00000000000..222903b348e --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -0,0 +1,114 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents a simplified path to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + class Path + attr_reader :path, :universe + attr_accessor :name + + def initialize(path, universe, metadata = []) + @path = path + @universe = universe + @metadata = metadata + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + end + + def directory? + @path.end_with?('/') || @path.blank? + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + new(@path.chomp(basename)) + end + + def basename + directory? ? name + ::File::SEPARATOR : name + end + + def name + @name || @path.split(::File::SEPARATOR).last + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]+/?$} + @children = select { |entry| entry =~ child_pattern } + end + + def directories + return [] unless directory? + children.select(&:directory?) + end + + def directories! + return directories unless has_parent? + + dotted_parent = parent + dotted_parent.name = '..' + directories.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @index ||= @universe.index(@path) + @metadata[@index] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def exists? + @path.blank? || @universe.include?(@path) + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @universe == other.universe + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def new(path) + self.class.new(path, @universe, @metadata) + end + + def select + selected = @universe.select { |entry| yield entry } + selected.map { |path| new(path) } + end + end + end + end +end diff --git a/lib/gitlab/string_path.rb b/lib/gitlab/string_path.rb deleted file mode 100644 index 774d4244a2a..00000000000 --- a/lib/gitlab/string_path.rb +++ /dev/null @@ -1,139 +0,0 @@ -module Gitlab - ## - # Class that represents a simplified path to a file or directory - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - class StringPath - attr_reader :path, :universe - attr_accessor :name - - def initialize(path, universe, metadata = []) - @path = sanitize(path) - @universe = universe.map { |entry| sanitize(entry) } - @metadata = metadata - end - - def to_s - @path - end - - def absolute? - @path.start_with?('/') - end - - def relative? - !absolute? - end - - def directory? - @path.end_with?('/') - end - - def file? - !directory? - end - - def has_parent? - nodes > 1 - end - - def parent - return nil unless has_parent? - new(@path.sub(basename, '')) - end - - def basename - directory? ? name + ::File::SEPARATOR : name - end - - def name - @name || @path.split(::File::SEPARATOR).last - end - - def has_descendants? - descendants.any? - end - - def descendants - return [] unless directory? - select { |entry| entry =~ /^#{Regexp.escape(@path)}.+/ } - end - - def children - return [] unless directory? - return @children if @children - - @children = select do |entry| - entry =~ %r{^#{Regexp.escape(@path)}[^/\s]+/?$} - end - end - - def directories - return [] unless directory? - children.select(&:directory?) - end - - def directories! - return directories unless has_parent? && directory? - - dotted_parent = parent - dotted_parent.name = '..' - directories.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - index = @universe.index(@path) - @metadata[index] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def exists? - @path == './' || @universe.include?(@path) - end - - def ==(other) - @path == other.path && @universe == other.universe - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def new(path) - self.class.new(path, @universe, @metadata) - end - - def select - selected = @universe.select { |entry| yield entry } - selected.map { |path| new(path) } - end - - def sanitize(path) - self.class.sanitize(path) - end - - def self.sanitize(path) - # It looks like Pathname#new doesn't touch a file system, - # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe - - clean_path = Pathname.new(path).cleanpath.to_s - raise ArgumentError, 'Invalid path' if clean_path.start_with?('../') - - prefix = './' unless clean_path =~ %r{^[\.|/]} - suffix = '/' if path.end_with?('/') || ['.', '..'].include?(clean_path) - prefix.to_s + clean_path + suffix.to_s - end - end -end -- cgit v1.2.1 From 09a4a5aff8c53dd5930044ddbb285a95ef177d8a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 11 Jan 2016 09:57:03 +0100 Subject: Render only valid paths in artifacts metadata In this version we will support only relative paths in artifacts metadata. Support for absolute paths will be introduced later. --- lib/gitlab/ci/build/artifacts/metadata.rb | 10 +++++++--- lib/gitlab/ci/build/artifacts/metadata/path.rb | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 996b5d91ff2..91017f633a0 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -12,7 +12,6 @@ module Gitlab def initialize(file, path) @file, @path = file, path @full_version = read_version - @path << '/' unless path.end_with?('/') || path.empty? end def version @@ -43,14 +42,15 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] - child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} until gz.eof? do begin path = read_string(gz) meta = read_string(gz) - next unless path =~ child_pattern + next unless path =~ match_pattern + next unless path_valid?(path) paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) @@ -62,6 +62,10 @@ module Gitlab [paths, metadata] end + def path_valid?(path) + !(path.start_with?('/') || path =~ %r{\.?\./}) + end + def read_version gzip do|gz| version_string = read_string(gz) diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb index 222903b348e..80ead335d57 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -23,7 +23,7 @@ module Gitlab end def directory? - @path.end_with?('/') || @path.blank? + blank_node? || @path.end_with?('/') end def file? @@ -40,11 +40,11 @@ module Gitlab end def basename - directory? ? name + ::File::SEPARATOR : name + (directory? && !blank_node?) ? name + ::File::SEPARATOR : name end def name - @name || @path.split(::File::SEPARATOR).last + @name || @path.split(::File::SEPARATOR).last.to_s end def children @@ -83,7 +83,11 @@ module Gitlab end def exists? - @path.blank? || @universe.include?(@path) + blank_node? || @universe.include?(@path) + end + + def blank_node? + @path.empty? # "" is considered to be './' end def to_s -- cgit v1.2.1 From ffee05c242c87e004054b48747287c3160d1c19a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 11 Jan 2016 12:50:21 +0100 Subject: Improve invalid build artifacts metadata path matcher --- lib/gitlab/ci/build/artifacts/metadata.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 91017f633a0..2b17712cdbe 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -43,6 +43,7 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do begin @@ -50,7 +51,7 @@ module Gitlab meta = read_string(gz) next unless path =~ match_pattern - next unless path_valid?(path) + next if path =~ invalid_pattern paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) @@ -62,10 +63,6 @@ module Gitlab [paths, metadata] end - def path_valid?(path) - !(path.start_with?('/') || path =~ %r{\.?\./}) - end - def read_version gzip do|gz| version_string = read_string(gz) -- cgit v1.2.1 From 2be76355caa579d444c8e3c0d25563eb9778bfb2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 10:04:26 +0100 Subject: Support only valid UTF-8 paths in build artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 8 +++++--- lib/gitlab/ci/build/artifacts/metadata/path.rb | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 2b17712cdbe..d9c051be9f3 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -10,7 +10,8 @@ module Gitlab attr_reader :file, :path, :full_version def initialize(file, path) - @file, @path = file, path + @file = file + @path = path.force_encoding('ASCII-8BIT') @full_version = read_version end @@ -42,7 +43,7 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] - match_pattern = %r{^#{Regexp.escape(@path)}[^/\s]*/?$} + match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do @@ -51,11 +52,12 @@ module Gitlab meta = read_string(gz) next unless path =~ match_pattern + next unless path.force_encoding('UTF-8').valid_encoding? next if path =~ invalid_pattern paths.push(path) metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) - rescue JSON::ParserError + rescue JSON::ParserError, Encoding::CompatibilityError next end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb index 80ead335d57..6896aa936d5 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/path.rb @@ -8,18 +8,24 @@ module Gitlab # This is IO-operations safe class, that does similar job to # Ruby's Pathname but without the risk of accessing filesystem. # + # This class is working only with UTF-8 encoded paths. + # class Path attr_reader :path, :universe attr_accessor :name def initialize(path, universe, metadata = []) - @path = path + @path = path.force_encoding('UTF-8') @universe = universe @metadata = metadata if path.include?("\0") raise ArgumentError, 'Path contains zero byte character!' end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end end def directory? @@ -51,7 +57,7 @@ module Gitlab return [] unless directory? return @children if @children - child_pattern = %r{^#{Regexp.escape(@path)}[^/\s]+/?$} + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} @children = select { |entry| entry =~ child_pattern } end -- cgit v1.2.1 From 487b0a026f9efe2d8214c19a7b95b391708ba3f4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 11:02:15 +0100 Subject: Improvements, readability for artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d9c051be9f3..47efc51a76e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab next if path =~ invalid_pattern paths.push(path) - metadata.push(JSON.parse(meta.chomp, symbolize_names: true)) + metadata.push(JSON.parse(meta, symbolize_names: true)) rescue JSON::ParserError, Encoding::CompatibilityError next end @@ -66,7 +66,7 @@ module Gitlab end def read_version - gzip do|gz| + gzip do |gz| version_string = read_string(gz) unless version_string @@ -95,9 +95,11 @@ module Gitlab def gzip open do |file| gzip = Zlib::GzipReader.new(file) - result = yield gzip - gzip.close - result + begin + yield gzip + ensure + gzip.close + end end end -- cgit v1.2.1 From e8995f9fd5c12882eafcf3766627f64a3d6f623d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 12 Jan 2016 14:39:15 +0100 Subject: Modify artifacts upload API endpoint, add artifacts metadata --- lib/ci/api/builds.rb | 20 ++++++++++++++++---- lib/ci/api/entities.rb | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..2aaa06a022c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -78,11 +78,15 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file + # metadata (optional) - Artifacts metadata file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - real filename as send in Content-Disposition + # metadata.type - real content type as send in Content-Type # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -98,10 +102,17 @@ module Ci authenticate_build_token!(build) forbidden!('build is not running') unless build.running? - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file!(:file, artifacts_upload_path) + file_to_large! unless artifacts.size < max_artifacts_size + artifacts_attributes = { artifacts_file: artifacts } - if build.update_attributes(artifacts_file: file) + if params[:metadata] || params['metadata.path'.to_sym] + metadata = uploaded_file!(:metadata, artifacts_upload_path) + artifacts_attributes.store(:artifacts_metadata, metadata) + end + + if build.update_attributes(artifacts_attributes) present build, with: Entities::Build else render_validation_error!(build) @@ -148,6 +159,7 @@ module Ci not_found! unless build authenticate_build_token!(build) build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index e4ac0545ea2..dd34c661e25 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -31,6 +31,7 @@ module Ci expose :variables expose :artifacts_file, using: ArtifactFile + expose :artifacts_metadata, using: ArtifactFile end class Runner < Grape::Entity -- cgit v1.2.1 From 0b946029a1fb429db39fbec0cddccf40f7e2aa08 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 09:56:05 +0100 Subject: Update build artifacts API We do not want to allow runners to upload a metadata file. This needs to be generated by Workhorse only. --- lib/ci/api/builds.rb | 17 ++++++++--------- lib/ci/api/entities.rb | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2aaa06a022c..be2790571cb 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -79,14 +79,13 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file - # metadata (optional) - Artifacts metadata file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - real filename as send in Content-Disposition - # metadata.type - real content type as send in Content-Type + # metadata.name - filename (generated by Workhorse) + # metadata.type - content type (returned by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -101,19 +100,19 @@ module Ci not_found! unless build authenticate_build_token!(build) forbidden!('build is not running') unless build.running? + forbidden!('metadata reserved for workhorse') if params[:metadata] artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file!(:file, artifacts_upload_path) file_to_large! unless artifacts.size < max_artifacts_size - artifacts_attributes = { artifacts_file: artifacts } + build.artifacts_file = artifacts - if params[:metadata] || params['metadata.path'.to_sym] - metadata = uploaded_file!(:metadata, artifacts_upload_path) - artifacts_attributes.store(:artifacts_metadata, metadata) + if params[:'metadata.path'] && params[:'metadata.name'] + build.artifacts_metadata = uploaded_file!(:metadata, artifacts_upload_path) end - if build.update_attributes(artifacts_attributes) - present build, with: Entities::Build + if build.save + present(build, with: Entities::Build) else render_validation_error!(build) end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index dd34c661e25..e4ac0545ea2 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -31,7 +31,6 @@ module Ci expose :variables expose :artifacts_file, using: ArtifactFile - expose :artifacts_metadata, using: ArtifactFile end class Runner < Grape::Entity -- cgit v1.2.1 From a9783c439bd6e3322b6fc72371c9fe3837a63be5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 10:13:39 +0100 Subject: Make encoding of paths returned by metadata consistent (UTF-8) --- lib/gitlab/ci/build/artifacts/metadata.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 47efc51a76e..0c252c0bf30 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -20,7 +20,7 @@ module Gitlab end def errors - gzip do|gz| + gzip do |gz| read_string(gz) # version errors = read_string(gz) raise StandardError, 'Errors field not found!' unless errors @@ -36,7 +36,7 @@ module Gitlab end def to_path - Path.new(@path, *match!) + Path.new(@path.dup.force_encoding('UTF-8'), *match!) end private @@ -88,7 +88,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) - return false unless string_size + return nil unless string_size gz.read(string_size) end -- cgit v1.2.1 From 3f0c18f80e36561581ef6fa6dbfcec169e1a6e08 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 13:45:28 +0100 Subject: Simplify encoding related implementation in artifacts metadata --- lib/gitlab/ci/build/artifacts/metadata.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 0c252c0bf30..e9ec8f1302c 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -10,8 +10,7 @@ module Gitlab attr_reader :file, :path, :full_version def initialize(file, path) - @file = file - @path = path.force_encoding('ASCII-8BIT') + @file, @path = file, path @full_version = read_version end @@ -36,7 +35,7 @@ module Gitlab end def to_path - Path.new(@path.dup.force_encoding('UTF-8'), *match!) + Path.new(@path, *match!) end private @@ -48,11 +47,11 @@ module Gitlab until gz.eof? do begin - path = read_string(gz) - meta = read_string(gz) + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern - next unless path.force_encoding('UTF-8').valid_encoding? next if path =~ invalid_pattern paths.push(path) -- cgit v1.2.1 From 154b8ceba4ac2d92a2387ad50d7f2b4ed5b2dd8a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 14:02:36 +0100 Subject: Refactor build artifacts upload API endpoint --- lib/api/helpers.rb | 4 +++- lib/ci/api/builds.rb | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..d46b5c42967 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -289,12 +289,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index be2790571cb..fb87637b94f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -85,7 +85,6 @@ module Ci # file.type - real content type as send in Content-Type # metadata.path - path to locally stored body (generated by Workhorse) # metadata.name - filename (generated by Workhorse) - # metadata.type - content type (returned by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -99,17 +98,17 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? - forbidden!('metadata reserved for workhorse') if params[:metadata] + forbidden!('Build is not running!') unless build.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file!(:file, artifacts_upload_path) + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) + + bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - build.artifacts_file = artifacts - if params[:'metadata.path'] && params[:'metadata.name'] - build.artifacts_metadata = uploaded_file!(:metadata, artifacts_upload_path) - end + build.artifacts_file = artifacts + build.artifacts_metadata = metadata if build.save present(build, with: Entities::Build) -- cgit v1.2.1 From 6b0a43aff36f0bbb9050b3c04155a3ccd9c1a75b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 21:17:28 +0100 Subject: Improve readability of artifacts browser `Entry` related code --- lib/gitlab/ci/build/artifacts/metadata.rb | 5 +- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 125 ++++++++++++++++++++++++ lib/gitlab/ci/build/artifacts/metadata/path.rb | 124 ----------------------- 3 files changed, 128 insertions(+), 126 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/metadata/entry.rb delete mode 100644 lib/gitlab/ci/build/artifacts/metadata/path.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index e9ec8f1302c..bfdfc9a1d7d 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -34,8 +34,9 @@ module Gitlab end end - def to_path - Path.new(@path, *match!) + def to_entry + entires, metadata = match! + Entry.new(@path, entires, metadata) end private diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..2fb6c327729 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,125 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entires + attr_accessor :name + + def initialize(path, entires, metadata = []) + @path = path.force_encoding('UTF-8') + @entires = entires + @metadata = metadata + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + new(@path.chomp(basename)) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entires { |entry| entry =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @index ||= @entires.index(@path) + @metadata[@index] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entires.include?(@path) + end + + def empty? + children.empty? + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entires == other.entires + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def new(path) + self.class.new(path, @entires, @metadata) + end + + def select_entires + selected = @entires.select { |entry| yield entry } + selected.map { |path| new(path) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/path.rb b/lib/gitlab/ci/build/artifacts/metadata/path.rb deleted file mode 100644 index 6896aa936d5..00000000000 --- a/lib/gitlab/ci/build/artifacts/metadata/path.rb +++ /dev/null @@ -1,124 +0,0 @@ -module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents a simplified path to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Path - attr_reader :path, :universe - attr_accessor :name - - def initialize(path, universe, metadata = []) - @path = path.force_encoding('UTF-8') - @universe = universe - @metadata = metadata - - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' - end - - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' - end - end - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - new(@path.chomp(basename)) - end - - def basename - (directory? && !blank_node?) ? name + ::File::SEPARATOR : name - end - - def name - @name || @path.split(::File::SEPARATOR).last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select { |entry| entry =~ child_pattern } - end - - def directories - return [] unless directory? - children.select(&:directory?) - end - - def directories! - return directories unless has_parent? - - dotted_parent = parent - dotted_parent.name = '..' - directories.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @index ||= @universe.index(@path) - @metadata[@index] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def exists? - blank_node? || @universe.include?(@path) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @universe == other.universe - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def new(path) - self.class.new(path, @universe, @metadata) - end - - def select - selected = @universe.select { |entry| yield entry } - selected.map { |path| new(path) } - end - end - end - end -end -- cgit v1.2.1 From ad2b0358e0facd5c65c4141ce54c2e55bab165e6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 22:31:27 +0100 Subject: Improve readability of artifacts `Metadata` related code --- lib/gitlab/ci/build/artifacts/metadata.rb | 32 +++++++++---------------- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 26 ++++++++++---------- 2 files changed, 24 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index bfdfc9a1d7d..94821c0eae0 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,7 +6,9 @@ module Gitlab module Build module Artifacts class Metadata - VERSION_PATTERN = '[\w\s]+(\d+\.\d+\.\d+)' + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + attr_reader :file, :path, :full_version def initialize(file, path) @@ -15,7 +17,7 @@ module Gitlab end def version - @full_version.match(/#{VERSION_PATTERN}/).captures.first + @full_version.match(VERSION_PATTERN)[1] end def errors @@ -27,7 +29,7 @@ module Gitlab end end - def match! + def find_entries! gzip do |gz| 2.times { read_string(gz) } # version and errors fields match_entries(gz) @@ -35,8 +37,8 @@ module Gitlab end def to_entry - entires, metadata = match! - Entry.new(@path, entires, metadata) + entries, metadata = find_entries! + Entry.new(@path, entries, metadata) end private @@ -44,7 +46,6 @@ module Gitlab def match_entries(gz) paths, metadata = [], [] match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} - invalid_pattern = %r{(^\.?\.?/)|(/\.?\.?/)} until gz.eof? do begin @@ -53,7 +54,7 @@ module Gitlab next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern - next if path =~ invalid_pattern + next if path =~ INVALID_PATH_PATTERN paths.push(path) metadata.push(JSON.parse(meta, symbolize_names: true)) @@ -73,7 +74,7 @@ module Gitlab raise StandardError, 'Artifacts metadata file empty!' end - unless version_string =~ /^#{VERSION_PATTERN}/ + unless version_string =~ VERSION_PATTERN raise StandardError, 'Invalid version!' end @@ -92,19 +93,8 @@ module Gitlab gz.read(string_size) end - def gzip - open do |file| - gzip = Zlib::GzipReader.new(file) - begin - yield gzip - ensure - gzip.close - end - end - end - - def open - File.open(@file) { |file| yield file } + def gzip(&block) + Zlib::GzipReader.open(@file, &block) end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2fb6c327729..12bb1bf0346 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -11,12 +11,12 @@ module Gitlab # This class is working only with UTF-8 encoded paths. # class Entry - attr_reader :path, :entires + attr_reader :path, :entries attr_accessor :name - def initialize(path, entires, metadata = []) + def initialize(path, entries, metadata = []) @path = path.force_encoding('UTF-8') - @entires = entires + @entries = entries @metadata = metadata if path.include?("\0") @@ -42,7 +42,7 @@ module Gitlab def parent return nil unless has_parent? - new(@path.chomp(basename)) + new_entry(@path.chomp(basename)) end def basename @@ -58,7 +58,7 @@ module Gitlab return @children if @children child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entires { |entry| entry =~ child_pattern } + @children = select_entries { |entry| entry =~ child_pattern } end def directories(opts = {}) @@ -77,7 +77,7 @@ module Gitlab end def metadata - @index ||= @entires.index(@path) + @index ||= @entries.index(@path) @metadata[@index] || {} end @@ -90,7 +90,7 @@ module Gitlab end def exists? - blank_node? || @entires.include?(@path) + blank_node? || @entries.include?(@path) end def empty? @@ -102,7 +102,7 @@ module Gitlab end def ==(other) - @path == other.path && @entires == other.entires + @path == other.path && @entries == other.entries end def inspect @@ -111,13 +111,13 @@ module Gitlab private - def new(path) - self.class.new(path, @entires, @metadata) + def new_entry(path) + self.class.new(path, @entries, @metadata) end - def select_entires - selected = @entires.select { |entry| yield entry } - selected.map { |path| new(path) } + def select_entries + selected = @entries.select { |entry| yield entry } + selected.map { |path| new_entry(path) } end end end -- cgit v1.2.1 From 0d6e7b9d3d38e60e5a706956a853e7dc940e4574 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 13 Jan 2016 23:24:28 +0100 Subject: Use Hash to store paths and entries metadata in artifacts browser --- lib/gitlab/ci/build/artifacts/metadata.rb | 11 +++++------ lib/gitlab/ci/build/artifacts/metadata/entry.rb | 18 ++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 94821c0eae0..25ecf27d4e6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -37,14 +37,14 @@ module Gitlab end def to_entry - entries, metadata = find_entries! - Entry.new(@path, entries, metadata) + entries = find_entries! + Entry.new(@path, entries) end private def match_entries(gz) - paths, metadata = [], [] + entries = {} match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} until gz.eof? do @@ -56,14 +56,13 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - paths.push(path) - metadata.push(JSON.parse(meta, symbolize_names: true)) + entries.store(path, JSON.parse(meta, symbolize_names: true)) rescue JSON::ParserError, Encoding::CompatibilityError next end end - [paths, metadata] + entries end def read_version diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 12bb1bf0346..4dae02ce4f7 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -14,10 +14,9 @@ module Gitlab attr_reader :path, :entries attr_accessor :name - def initialize(path, entries, metadata = []) - @path = path.force_encoding('UTF-8') + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') @entries = entries - @metadata = metadata if path.include?("\0") raise ArgumentError, 'Path contains zero byte character!' @@ -42,7 +41,7 @@ module Gitlab def parent return nil unless has_parent? - new_entry(@path.chomp(basename)) + self.class.new(@path.chomp(basename), @entries) end def basename @@ -77,8 +76,7 @@ module Gitlab end def metadata - @index ||= @entries.index(@path) - @metadata[@index] || {} + @entries[@path] || {} end def nodes @@ -111,13 +109,9 @@ module Gitlab private - def new_entry(path) - self.class.new(path, @entries, @metadata) - end - def select_entries - selected = @entries.select { |entry| yield entry } - selected.map { |path| new_entry(path) } + selected = @entries.select { |entry, _metadata| yield entry } + selected.map { |path, _metadata| self.class.new(path, @entries) } end end end -- cgit v1.2.1 From be764a3a20c7cecce2a047ddd46aff954c33b306 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Jan 2016 12:12:05 +0100 Subject: Minor improvements in build arfifacts browser Added also a `Gitlab::Ci::Build::Artifacts::Metadata::ParserError` exception class. --- lib/gitlab/ci/build/artifacts/metadata.rb | 17 ++++++++++++----- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 25ecf27d4e6..1344f5d120b 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,6 +6,8 @@ module Gitlab module Build module Artifacts class Metadata + class ParserError < StandardError; end + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} @@ -24,8 +26,13 @@ module Gitlab gzip do |gz| read_string(gz) # version errors = read_string(gz) - raise StandardError, 'Errors field not found!' unless errors - JSON.parse(errors) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end end end @@ -56,7 +63,7 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - entries.store(path, JSON.parse(meta, symbolize_names: true)) + entries[path] = JSON.parse(meta, symbolize_names: true) rescue JSON::ParserError, Encoding::CompatibilityError next end @@ -70,11 +77,11 @@ module Gitlab version_string = read_string(gz) unless version_string - raise StandardError, 'Artifacts metadata file empty!' + raise ParserError, 'Artifacts metadata file empty!' end unless version_string =~ VERSION_PATTERN - raise StandardError, 'Invalid version!' + raise ParserError, 'Invalid version!' end version_string.chomp diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 4dae02ce4f7..25b71fc3275 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -57,7 +57,7 @@ module Gitlab return @children if @children child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |entry| entry =~ child_pattern } + @children = select_entries { |path| path =~ child_pattern } end def directories(opts = {}) @@ -110,7 +110,7 @@ module Gitlab private def select_entries - selected = @entries.select { |entry, _metadata| yield entry } + selected = @entries.select { |path, _metadata| yield path } selected.map { |path, _metadata| self.class.new(path, @entries) } end end -- cgit v1.2.1 From 05187f0fc671e1ddcbe6d7b0f76971d792acaada Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 13:09:36 +0100 Subject: Properly generate diff of orphan commits, like the first commit in a repository --- lib/gitlab/diff/parser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..516e59b87a3 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -56,8 +56,9 @@ module Gitlab private def filename?(line) - line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', - '--- /tmp/diffy', '+++ /tmp/diffy') + line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit + '--- /tmp/diffy', '+++ /tmp/diffy') end def identification_type(line) -- cgit v1.2.1 From c31d777c8f24029d7e11dd1e78eddf0c0b6e6f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 14 Jan 2016 14:20:21 +0100 Subject: Revert changes to how the notes are paginated in the API --- lib/api/notes.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index ebd9e97148c..174473f5371 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -22,11 +22,17 @@ module API @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) # We exclude notes that are cross-references and that cannot be viewed - # by the current user. + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. notes = - @noteable.notes. + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). reject { |n| n.cross_reference_not_visible_for?(current_user) } - present paginate(Kaminari.paginate_array(notes)), with: Entities::Note + present notes, with: Entities::Note end # Get a single +noteable+ note -- cgit v1.2.1 From 2c7d9cfa7dbd4e7716793bdb1ee9e081f13c33b2 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 14 Jan 2016 14:59:04 +0100 Subject: Move Ci::Build#available_statuses to AVAILABLE_STATUSES constant in CommitStatus --- lib/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 1337e1bb452..d293f988165 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -125,7 +125,7 @@ module API def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? - available_statuses = Ci::Build.available_statuses + available_statuses = ::CommitStatus::AVAILABLE_STATUSES scope = if scope.is_a?(String) [scope] -- cgit v1.2.1 From 78f5eb94fb15f7e4cc4208a4871b9533243bec40 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 6 Jan 2016 21:15:37 -0200 Subject: Import GitHub wiki into GitLab --- lib/gitlab/github_import/importer.rb | 13 +++++++++++++ lib/gitlab/github_import/project_creator.rb | 3 ++- lib/gitlab/github_import/wiki_formatter.rb | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/github_import/wiki_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2b0afbc7b39..f75227b4734 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -14,8 +16,11 @@ module Gitlab def execute import_issues import_pull_requests + import_wiki true + rescue Gitlab::Shell::Error + false end private @@ -66,6 +71,14 @@ module Gitlab noteable.notes.create!(comment.attributes) end end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6dd2b917c4a --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(".git", ".wiki.git") + end + end + end +end -- cgit v1.2.1 From a6a5990ee5f504107944c3bba5c18dbdea9f5207 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:05:18 -0200 Subject: Add Banzai::Filter::GollumTagsFilter for parsing Gollum's tags in HTML --- lib/banzai/filter/gollum_tags_filter.rb | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 lib/banzai/filter/gollum_tags_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb new file mode 100644 index 00000000000..f78b30bd4e9 --- /dev/null +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -0,0 +1,106 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + # HTML Filter for parsing Gollum's tags in HTML. + # + # Based on Gollum::Filter::Tags + # + # Context options: + # :project_wiki (required) - Current project wiki. + # + class GollumTagsFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match tag contents. + TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + + def call + search_text_nodes(doc).each do |node| + content = node.content + + next unless content.match(TAGS_PATTERN) + + html = process_tag($2) + + node.replace(html) if html != node.content + end + + doc + end + + private + + # Process a single tag into its final HTML form. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML version of the tag. + def process_tag(tag) + if html = process_image_tag(tag) + html + else + process_page_link_tag(tag) + end + end + + # Attempt to process the tag as an image tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid image tag or nil + # if it is not. + def process_image_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + name = parts[0].strip + + if file = project_wiki.find_file(name) + path = ::File.join project_wiki_base_path, file.path + elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i + path = name + end + + if path + content_tag(:img, nil, src: path) + end + end + + # Attempt to process the tag as a page link tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid page link tag or nil + # if it is not. + def process_page_link_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + if parts.size == 1 + url = parts[0].strip + else + name, url = *parts.compact.map(&:strip) + end + + content_tag(:a, name || url, href: url) + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + + # Ensure that a :project_wiki key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project_wiki + end + end + end +end -- cgit v1.2.1 From 4872b319c8152837bd36e7fc0a5ad912d1c3ef90 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:10:08 -0200 Subject: Use the WikiPipeline when rendering the wiki markdown content --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 +++++++++++++ lib/banzai/pipeline/wiki_pipeline.rb | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb create mode 100644 lib/banzai/pipeline/wiki_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb new file mode 100644 index 00000000000..1f98d3a183a --- /dev/null +++ b/lib/banzai/pipeline/gollum_tags_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class GollumTagsPipeline < BasePipeline + def self.filters + [ + Filter::GollumTagsFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb new file mode 100644 index 00000000000..4635a8d6471 --- /dev/null +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -0,0 +1,9 @@ +require 'banzai' + +module Banzai + module Pipeline + class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) + + end + end +end -- cgit v1.2.1 From 60f8434a9d7def80e43ca2e1c8e4e3e502856913 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 12:41:22 -0200 Subject: Refactoring Gitlab::GithubImport::Importer --- lib/gitlab/github_import/importer.rb | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index f75227b4734..18929b9113b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -14,13 +14,7 @@ module Gitlab end def execute - import_issues - import_pull_requests - import_wiki - - true - rescue Gitlab::Shell::Error - false + import_issues && import_pull_requests && import_wiki end private @@ -39,6 +33,10 @@ module Gitlab end end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_pull_requests @@ -53,6 +51,10 @@ module Gitlab import_comments_on_diff(pull_request.number, merge_request) end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_comments(issue_number, noteable) @@ -78,6 +80,10 @@ module Gitlab gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end + + true + rescue Gitlab::Shell::Error + false end end end -- cgit v1.2.1 From ca87bb652a1a8390cdc686059f670b89b274dce2 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:16:32 -0200 Subject: Remove GollumTagsPipeline --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 ------------- lib/banzai/pipeline/wiki_pipeline.rb | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb deleted file mode 100644 index 1f98d3a183a..00000000000 --- a/lib/banzai/pipeline/gollum_tags_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'banzai' - -module Banzai - module Pipeline - class GollumTagsPipeline < BasePipeline - def self.filters - [ - Filter::GollumTagsFilter - ] - end - end - end -end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 4635a8d6471..50b5450e70b 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -2,8 +2,10 @@ require 'banzai' module Banzai module Pipeline - class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) - + class WikiPipeline < FullPipeline + def self.filters + super.insert(1, Filter::GollumTagsFilter) + end end end end -- cgit v1.2.1 From 89e8b82b638e440bc487c4135cdfa4402fdffde5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:50:25 -0200 Subject: Make sure the .git is at the end on Gitlab::GithubImport::WikiFormatter --- lib/gitlab/github_import/wiki_formatter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6dd2b917c4a..6c592ff469c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -12,7 +12,7 @@ module Gitlab end def import_url - project.import_url.sub(".git", ".wiki.git") + project.import_url.sub(/\.git\z/, ".wiki.git") end end end -- cgit v1.2.1 From 765a2c73271cf311311c391e7e64f83e141c79ae Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 18:59:49 -0200 Subject: Refactoring Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 47 ++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f78b30bd4e9..e4a73030597 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -14,7 +14,10 @@ module Banzai include ActionView::Helpers::TagHelper # Pattern to match tag contents. - TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + TAGS_PATTERN = %r{\[\[(.+?)\]\]} + + # Pattern to match allowed image extensions + ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i def call search_text_nodes(doc).each do |node| @@ -22,9 +25,11 @@ module Banzai next unless content.match(TAGS_PATTERN) - html = process_tag($2) + html = process_tag($1) - node.replace(html) if html != node.content + if html && html != node.content + node.replace(html) + end end doc @@ -38,11 +43,11 @@ module Banzai # # Returns the String HTML version of the tag. def process_tag(tag) - if html = process_image_tag(tag) - html - else - process_page_link_tag(tag) - end + parts = tag.split('|') + + return if parts.size.zero? + + process_image_tag(parts) || process_page_link_tag(parts) end # Attempt to process the tag as an image tag. @@ -51,16 +56,15 @@ module Banzai # # Returns the String HTML if the tag is a valid image tag or nil # if it is not. - def process_image_tag(tag) - parts = tag.split('|') - return if parts.size.zero? + def process_image_tag(parts) + content = parts[0].strip - name = parts[0].strip + return unless image?(content) - if file = project_wiki.find_file(name) + if url?(content) + path = content + elsif file = project_wiki.find_file(content) path = ::File.join project_wiki_base_path, file.path - elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i - path = name end if path @@ -68,16 +72,21 @@ module Banzai end end + def image?(path) + path =~ ALLOWED_IMAGE_EXTENSIONS + end + + def url?(path) + path.start_with?(*%w(http https)) + end + # Attempt to process the tag as a page link tag. # # tag - The String tag contents (the stuff inside the double brackets). # # Returns the String HTML if the tag is a valid page link tag or nil # if it is not. - def process_page_link_tag(tag) - parts = tag.split('|') - return if parts.size.zero? - + def process_page_link_tag(parts) if parts.size == 1 url = parts[0].strip else -- cgit v1.2.1 From ac2c86055eb189690bf67cf97cc6eb5ec9c2be7b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 20:24:17 -0200 Subject: Update documentation on Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 40 +++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index e4a73030597..fe01dae4850 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -3,7 +3,28 @@ require 'html/pipeline/filter' module Banzai module Filter - # HTML Filter for parsing Gollum's tags in HTML. + # HTML Filter for parsing Gollum's tags in HTML. It's only parses the + # following tags: + # + # - Link to internal pages: + # + # * [[Bug Reports]] + # * [[How to Contribute|Contributing]] + # + # - Link to external resources: + # + # * [[http://en.wikipedia.org/wiki/Git_(software)]] + # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]] + # + # - Link internal images, the special attributes will be ignored: + # + # * [[images/logo.png]] + # * [[images/logo.png|alt=Logo]] + # + # - Link external images, the special attributes will be ignored: + # + # * [[http://example.com/images/logo.png]] + # * [[http://example.com/images/logo.png|alt=Logo]] # # Based on Gollum::Filter::Tags # @@ -13,7 +34,22 @@ module Banzai class GollumTagsFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper - # Pattern to match tag contents. + # Pattern to match tags content that should be parsed in HTML. + # + # Gollum's tags have been made to resemble the tags of other markups, + # especially MediaWiki. The basic syntax is: + # + # [[tag]] + # + # Some tags will accept attributes which are separated by pipe + # symbols.Some attributes must precede the tag and some must follow it: + # + # [[prefix-attribute|tag]] + # [[tag|suffix-attribute]] + # + # See https://github.com/gollum/gollum/wiki + # + # Rubular: http://rubular.com/r/7dQnE5CUCH TAGS_PATTERN = %r{\[\[(.+?)\]\]} # Pattern to match allowed image extensions -- cgit v1.2.1 From 3a1d0535992594bc77320f081d1f20b760b1c1f7 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 15:11:50 +0100 Subject: Remove duplication around highlighting. --- lib/gitlab/diff/highlight.rb | 13 +------------ lib/gitlab/highlight.rb | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 lib/gitlab/highlight.rb (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index e76a6f27856..f34eff62d79 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -20,18 +20,7 @@ module Gitlab blob = repository.blob_at(ref, file_name) return [] unless blob - content = blob.data - lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new - formatter.format(lexer.lex(content)).lines.map!(&:html_safe) - end - - def self.formatter - @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + Gitlab::Highlight.highlight(file_name, blob.data).lines.map!(&:html_safe) end def initialize(diff_file) diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb new file mode 100644 index 00000000000..02e097eca3d --- /dev/null +++ b/lib/gitlab/highlight.rb @@ -0,0 +1,23 @@ +module Gitlab + class Highlight + def self.highlight(blob_name, blob_content, nowrap: true, continue: false) + formatter = rouge_formatter(nowrap: nowrap) + + lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + formatter.format(lexer.lex(blob_content, continue: continue)).html_safe + end + + private + + def self.rouge_formatter(options = {}) + options = options.reverse_merge( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + + Rouge::Formatters::HTMLGitlab.new(options) + end + end +end -- cgit v1.2.1 From 83e4fc188b22731d89106b4da28f11bf5509c116 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 16:13:35 +0100 Subject: Refactor highlighting lines --- lib/gitlab/diff/highlight.rb | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f34eff62d79..ba2f12db147 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -32,23 +32,8 @@ module Gitlab def highlight return [] if @lines.empty? - extract_line_prefixes - update_diff_lines - end - - private - - def text_lines - @text_lines ||= @lines.map(&:text) - end - - def extract_line_prefixes - @diff_line_prefixes ||= text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } - end - - def update_diff_lines @lines.each_with_index do |line, i| - line_prefix = @diff_line_prefixes[i] || ' ' + line_prefix = line.text.match(/\A([+-])/) ? $1 : ' ' # ignore highlighting for "match" lines next if line.type == 'match' @@ -62,24 +47,18 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.sub(/\A\s/, line_prefix).html_safe if highlighted_line + line.text = highlighted_line.insert(0, line_prefix).html_safe if highlighted_line end @lines end def old_lines - @old_lines ||= begin - lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) - lines.map! { |line| " #{line}" } - end + @old_lines ||= self.class.process_file(diff_repository, diff_old_ref, diff_old_path) end def new_lines - @new_lines ||= begin - lines = self.class.process_file(diff_repository, diff_new_ref, diff_new_path) - lines.map! { |line| " #{line}" } - end + @new_lines ||= self.class.process_file(diff_repository, diff_new_ref, diff_new_path) end end end -- cgit v1.2.1 From ac652d82f17d378e485dcef15a8fabdcf9bad76b Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 14 Jan 2016 19:45:43 +0100 Subject: Let the CI runner know about builds that this build depends on This allows us to implement artifacts passing: runner will download artifacts from all prior builds --- lib/ci/api/builds.rb | 4 ++-- lib/ci/api/entities.rb | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index fb87637b94f..690bbf97a89 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -20,7 +20,7 @@ module Ci if build update_runner_info - present build, with: Entities::Build + present build, with: Entities::BuildDetails else not_found! end @@ -111,7 +111,7 @@ module Ci build.artifacts_metadata = metadata if build.save - present(build, with: Entities::Build) + present(build, with: Entities::BuildDetails) else render_validation_error!(build) end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index e4ac0545ea2..835f8c97021 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -16,10 +16,19 @@ module Ci end class Build < Grape::Entity - expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, - :before_sha, :allow_git_fetch, :project_name - + expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage + expose :project_id + expose :project_name + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts_file.exists? } + end + + class BuildDetails < Build + expose :commands + expose :repo_url + expose :before_sha + expose :allow_git_fetch + expose :token expose :options do |model| model.options @@ -30,7 +39,9 @@ module Ci end expose :variables - expose :artifacts_file, using: ArtifactFile + expose :dependencies do + expose :depends_on_builds, as: :builds, using: Build + end end class Runner < Grape::Entity -- cgit v1.2.1 From 8dfad143d44af4896ff6c71e8a42ad32b69ad593 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 22:28:07 +0100 Subject: Add inline diff markers in highlighted diffs. --- lib/gitlab/diff/file.rb | 2 +- lib/gitlab/diff/highlight.rb | 206 ++++++++++++++++++++++++++++++++++++------- lib/gitlab/highlight.rb | 7 ++ 3 files changed, 181 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index cb93c6a574a..c1a6e16da5a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -18,7 +18,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(self) + Gitlab::Diff::Highlight.new(self).highlight end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index ba2f12db147..e21f496102d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -6,59 +6,199 @@ module Gitlab delegate :repository, :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - # Apply syntax highlight to provided source code - # - # diff_file - an instance of Gitlab::Diff::File - # - # Returns an Array with the processed items. - def self.process_diff_lines(diff_file) - processor = new(diff_file) - processor.highlight + def initialize(diff_file) + @diff_file = diff_file + @diff_lines = diff_file.diff_lines + @raw_lines = @diff_lines.map(&:text) end - def self.process_file(repository, ref, file_name) - blob = repository.blob_at(ref, file_name) - return [] unless blob + def highlight + return [] if @diff_lines.empty? - Gitlab::Highlight.highlight(file_name, blob.data).lines.map!(&:html_safe) - end + find_inline_diffs - def initialize(diff_file) - @diff_file = diff_file - @file_name = diff_file.new_path - @lines = diff_file.diff_lines + process_lines + + @diff_lines end - def highlight - return [] if @lines.empty? + private - @lines.each_with_index do |line, i| - line_prefix = line.text.match(/\A([+-])/) ? $1 : ' ' + def find_inline_diffs + @inline_diffs = [] + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = @raw_lines[old_index][1..-1] + new_line = @raw_lines[new_index][1..-1] + + # Skip inline diff if empty line was replaced with content + next if old_line == "" + + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line, new_line) + + old_diff_range = lcp..(old_line.length - lcs - 1) + new_diff_range = lcp..(new_line.length - lcs - 1) + + @inline_diffs[old_index] = old_diff_range if old_diff_range.begin <= old_diff_range.end + @inline_diffs[new_index] = new_diff_range if new_diff_range.begin <= new_diff_range.end + end + end + + def process_lines + @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines - next if line.type == 'match' + next if diff_line.type == 'match' + + rich_line = highlight_line(diff_line, i) + rich_line = mark_inline_diffs(rich_line, diff_line, i) + diff_line.text = rich_line.html_safe + end + end + + def highlight_line(diff_line, index) + line_prefix = line_prefixes[index] + + case diff_line.type + when 'new', nil + rich_line = new_lines[diff_line.new_pos - 1] + when 'old' + rich_line = old_lines[diff_line.old_pos - 1] + end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + rich_line ? line_prefix + rich_line : diff_line.text + end + + def mark_inline_diffs(rich_line, diff_line, index) + inline_diff = @inline_diffs[index] + return rich_line unless inline_diff + + raw_line = diff_line.text + + # Based on the prefixless versions + from = inline_diff.begin + 1 + to = inline_diff.end + 1 + + position_mapping = map_character_positions(raw_line, rich_line) + inline_diff_positions = position_mapping[from..to] + marker_ranges = collapse_ranges(inline_diff_positions) + + offset = 0 + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) + end + + rich_line + end - case line.type - when 'new', nil - highlighted_line = new_lines[line.new_pos - 1] - when 'old' - highlighted_line = old_lines[line.old_pos - 1] + def line_prefixes + @line_prefixes ||= @raw_lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + end + + def local_edit_indexes + @local_edit_indexes ||= begin + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 end - # Only update text if line is found. This will prevent - # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.insert(0, line_prefix).html_safe if highlighted_line + local_edit_indexes end + end + + def map_character_positions(raw_line, rich_line) + mapping = [] + raw_pos = 0 + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + raw_char = raw_line[raw_pos] + rich_char = rich_line[rich_pos] + + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end - @lines + mapping[raw_pos] = rich_pos + + rich_pos += 1 + end + + mapping end def old_lines - @old_lines ||= self.class.process_file(diff_repository, diff_old_ref, diff_old_path) + @old_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_old_ref, diff_old_path) end def new_lines - @new_lines ||= self.class.process_file(diff_repository, diff_new_ref, diff_new_path) + @new_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_new_ref, diff_new_path) + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + def insert_around_range(text, range, before, after, offset = 0) + from = range.begin + to = range.end + + text.insert(offset + from, before) + offset += before.length + + text.insert(offset + to + 1, after) + offset += after.length + + offset end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 02e097eca3d..a5b041687e3 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -7,6 +7,13 @@ module Gitlab formatter.format(lexer.lex(blob_content, continue: continue)).html_safe end + def self.highlight_lines(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + highlight(file_name, blob.data).lines.map!(&:html_safe) + end + private def self.rouge_formatter(options = {}) -- cgit v1.2.1 From 6b9c730e91962a6d6343bcb7fc4dc75c99b41bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 14 Jan 2016 16:38:37 -0500 Subject: More refactoring from last code review. #3945 * Use commit objects instead of IDs when generating diffs * Use proper references when generating MR's source and target * Update broken specs --- lib/gitlab/diff/file.rb | 5 ++--- lib/gitlab/diff/highlight.rb | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c1a6e16da5a..a6a7fc8ff4c 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,14 +1,13 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :new_ref, :old_ref + attr_reader :diff, :new_ref, :old_ref delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff, diff_refs, repository) + def initialize(diff, diff_refs) @diff = diff - @repository = repository @old_ref, @new_ref = diff_refs end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index e21f496102d..fb79a2a69de 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,8 +3,7 @@ module Gitlab class Highlight attr_reader :diff_file - delegate :repository, :old_path, :new_path, :old_ref, :new_ref, - to: :diff_file, prefix: :diff + delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff def initialize(diff_file) @diff_file = diff_file @@ -141,11 +140,11 @@ module Gitlab end def old_lines - @old_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_old_ref, diff_old_path) + @old_lines ||= self.class.process_file(*processing_args(:old)) end def new_lines - @new_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_new_ref, diff_new_path) + @new_lines ||= self.class.process_file(*processing_args(:new)) end def longest_common_suffix(a, b) @@ -200,6 +199,16 @@ module Gitlab offset end + + private + + def processing_args(version) + ref = send("diff_#{version}_ref") + path = send("diff_#{version}_path") + + [ref.project.repository, ref.id, path] + end + end end end -- cgit v1.2.1 From 71b4341a37dc274ce94d0b967e5b69fd49834cb8 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:11:36 +0100 Subject: Method was moved --- lib/gitlab/diff/highlight.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index fb79a2a69de..65deea9d335 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -140,11 +140,11 @@ module Gitlab end def old_lines - @old_lines ||= self.class.process_file(*processing_args(:old)) + @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) end def new_lines - @new_lines ||= self.class.process_file(*processing_args(:new)) + @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end def longest_common_suffix(a, b) -- cgit v1.2.1 From 7d31f372191c38e5676cd6ec746721eb64b4a634 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:25:29 +0100 Subject: Move inline diff logic to its own class --- lib/gitlab/diff/highlight.rb | 84 ++++++-------------------------------------- 1 file changed, 11 insertions(+), 73 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 65deea9d335..f89121e184b 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,26 +24,7 @@ module Gitlab private def find_inline_diffs - @inline_diffs = [] - - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 - old_line = @raw_lines[old_index][1..-1] - new_line = @raw_lines[new_index][1..-1] - - # Skip inline diff if empty line was replaced with content - next if old_line == "" - - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line, new_line) - - old_diff_range = lcp..(old_line.length - lcs - 1) - new_diff_range = lcp..(new_line.length - lcs - 1) - - @inline_diffs[old_index] = old_diff_range if old_diff_range.begin <= old_diff_range.end - @inline_diffs[new_index] = new_diff_range if new_diff_range.begin <= new_diff_range.end - end + @inline_diffs = InlineDiff.new(@raw_lines).inline_diffs end def process_lines @@ -58,7 +39,7 @@ module Gitlab end def highlight_line(diff_line, index) - line_prefix = line_prefixes[index] + line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' case diff_line.type when 'new', nil @@ -73,44 +54,23 @@ module Gitlab end def mark_inline_diffs(rich_line, diff_line, index) - inline_diff = @inline_diffs[index] - return rich_line unless inline_diff + line_inline_diffs = @inline_diffs[index] + return rich_line unless line_inline_diffs raw_line = diff_line.text - - # Based on the prefixless versions - from = inline_diff.begin + 1 - to = inline_diff.end + 1 - position_mapping = map_character_positions(raw_line, rich_line) - inline_diff_positions = position_mapping[from..to] - marker_ranges = collapse_ranges(inline_diff_positions) offset = 0 - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end - - rich_line - end - - def line_prefixes - @line_prefixes ||= @raw_lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - end - - def local_edit_indexes - @local_edit_indexes ||= begin - joined_line_prefixes = " #{line_prefixes.join} " + line_inline_diffs.each do |inline_diff_range| + inline_diff_positions = position_mapping[inline_diff_range] + marker_ranges = collapse_ranges(inline_diff_positions) - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) end - - local_edit_indexes end + + rich_line end def map_character_positions(raw_line, rich_line) @@ -147,25 +107,6 @@ module Gitlab @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end - def longest_common_suffix(a, b) - longest_common_prefix(a.reverse, b.reverse) - end - - def longest_common_prefix(a, b) - max_length = [a.length, b.length].max - - length = 0 - (0..max_length - 1).each do |pos| - old_char = a[pos] - new_char = b[pos] - - break if old_char != new_char - length += 1 - end - - length - end - def collapse_ranges(positions) return [] if positions.empty? ranges = [] @@ -200,15 +141,12 @@ module Gitlab offset end - private - def processing_args(version) ref = send("diff_#{version}_ref") path = send("diff_#{version}_path") [ref.project.repository, ref.id, path] end - end end end -- cgit v1.2.1 From 13f10efcf120f2d5244bf6abe934dc7c026834ef Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:39:43 +0100 Subject: Move inline diff marker logic to its own class --- lib/gitlab/diff/highlight.rb | 106 +++++-------------------------------------- 1 file changed, 11 insertions(+), 95 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f89121e184b..b6875f07279 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -12,32 +12,24 @@ module Gitlab end def highlight - return [] if @diff_lines.empty? - - find_inline_diffs - - process_lines - - @diff_lines - end - - private - - def find_inline_diffs - @inline_diffs = InlineDiff.new(@raw_lines).inline_diffs - end - - def process_lines @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines next if diff_line.type == 'match' rich_line = highlight_line(diff_line, i) - rich_line = mark_inline_diffs(rich_line, diff_line, i) + + if line_inline_diffs = inline_diffs[i] + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + end + diff_line.text = rich_line.html_safe end + + @diff_lines end + private + def highlight_line(diff_line, index) line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' @@ -53,50 +45,8 @@ module Gitlab rich_line ? line_prefix + rich_line : diff_line.text end - def mark_inline_diffs(rich_line, diff_line, index) - line_inline_diffs = @inline_diffs[index] - return rich_line unless line_inline_diffs - - raw_line = diff_line.text - position_mapping = map_character_positions(raw_line, rich_line) - - offset = 0 - line_inline_diffs.each do |inline_diff_range| - inline_diff_positions = position_mapping[inline_diff_range] - marker_ranges = collapse_ranges(inline_diff_positions) - - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end - end - - rich_line - end - - def map_character_positions(raw_line, rich_line) - mapping = [] - raw_pos = 0 - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - raw_char = raw_line[raw_pos] - rich_char = rich_line[rich_pos] - - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = rich_pos - - rich_pos += 1 - end - - mapping + def inline_diffs + @inline_diffs ||= InlineDiff.new(@raw_lines).inline_diffs end def old_lines @@ -107,40 +57,6 @@ module Gitlab @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - def insert_around_range(text, range, before, after, offset = 0) - from = range.begin - to = range.end - - text.insert(offset + from, before) - offset += before.length - - text.insert(offset + to + 1, after) - offset += after.length - - offset - end - def processing_args(version) ref = send("diff_#{version}_ref") path = send("diff_#{version}_path") -- cgit v1.2.1 From 2444c04055c7cc7d48c0301e8cd9d30fc534d460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20K=C3=B6hler?= Date: Thu, 7 Jan 2016 19:14:41 +0100 Subject: Fix signup for some OAuth providers some OAuth providers (kerberos for example) only provide a username and an email, but no name. Therefore a signup fails because the name is empty. Best guess for the name is probably the username, therefore use it as name. --- lib/gitlab/o_auth/user.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f1a362f5303..e3d2cc65a8f 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -141,9 +141,12 @@ module Gitlab username = auth_hash.username email = auth_hash.email end + + name = auth_hash.name + name = ::Namespace.clean_path(username) if name.strip.empty? { - name: auth_hash.name, + name: name, username: ::Namespace.clean_path(username), email: email, password: auth_hash.password, -- cgit v1.2.1 From 1ce766367eb529fe88068be2f34315f87d74a349 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 15 Jan 2016 15:35:33 +0100 Subject: Change dependencies.builds to depends_on_builds --- lib/ci/api/entities.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 835f8c97021..b25e0e573a8 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts_file.exists? } + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } end class BuildDetails < Build @@ -39,9 +39,7 @@ module Ci end expose :variables - expose :dependencies do - expose :depends_on_builds, as: :builds, using: Build - end + expose :depends_on_builds, using: Build end class Runner < Grape::Entity -- cgit v1.2.1 From 5de8971ddfa10cbc6a082316eee7377038670f75 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 16:02:48 +0100 Subject: Whoops, forgot to add files --- lib/gitlab/diff/inline_diff.rb | 75 +++++++++++++++++++++++++++++++ lib/gitlab/diff/inline_diff_marker.rb | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 lib/gitlab/diff/inline_diff.rb create mode 100644 lib/gitlab/diff/inline_diff_marker.rb (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb new file mode 100644 index 00000000000..bae297ab00f --- /dev/null +++ b/lib/gitlab/diff/inline_diff.rb @@ -0,0 +1,75 @@ +module Gitlab + module Diff + class InlineDiff + attr_accessor :lines + + def initialize(lines) + @lines = lines + end + + def inline_diffs + inline_diffs = [] + + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = @lines[old_index] + new_line = @lines[new_index] + + suffixless_old_line = old_line[1..-1] + suffixless_new_line = new_line[1..-1] + + # Skip inline diff if empty line was replaced with content + next if suffixless_old_line == "" + + # Add one, because this is based on the suffixless version + lcp = longest_common_prefix(suffixless_old_line, suffixless_new_line) + 1 + lcs = longest_common_suffix(suffixless_old_line, suffixless_new_line) + + old_diff_range = lcp..(old_line.length - lcs - 1) + new_diff_range = lcp..(new_line.length - lcs - 1) + + inline_diffs[old_index] = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + inline_diffs[new_index] = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + end + + inline_diffs + end + + private + + def local_edit_indexes + line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 + end + + local_edit_indexes + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb new file mode 100644 index 00000000000..4bb755e3d3d --- /dev/null +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -0,0 +1,85 @@ +module Gitlab + module Diff + class InlineDiffMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = rich_line + end + + def mark(line_inline_diffs) + offset = 0 + line_inline_diffs.each do |inline_diff_range| + inline_diff_positions = position_mapping[inline_diff_range] + marker_ranges = collapse_ranges(inline_diff_positions) + + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) + end + end + + rich_line + end + + def position_mapping + @position_mapping ||= begin + mapping = [] + raw_pos = 0 + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + raw_char = raw_line[raw_pos] + rich_char = rich_line[rich_pos] + + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = rich_pos + + rich_pos += 1 + end + + mapping + end + end + + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + def insert_around_range(text, range, before, after, offset = 0) + text.insert(offset + range.begin, before) + offset += before.length + + text.insert(offset + range.end + 1, after) + offset += after.length + + offset + end + end + end +end -- cgit v1.2.1 From 12546f895f86929ffe69299a02c93692370ee55a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 16:29:26 +0100 Subject: Add tests --- lib/gitlab/diff/inline_diff.rb | 1 + lib/gitlab/diff/inline_diff_marker.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index bae297ab00f..e5986fd69e2 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -38,6 +38,7 @@ module Gitlab private + # Find runs of single line edits def local_edit_indexes line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } joined_line_prefixes = " #{line_prefixes.join} " diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 4bb755e3d3d..405465a641f 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -11,9 +11,12 @@ module Gitlab def mark(line_inline_diffs) offset = 0 line_inline_diffs.each do |inline_diff_range| + # Map the inline-diff range based on the raw line to character positions in the rich line inline_diff_positions = position_mapping[inline_diff_range] + # Turn the array of character positions into ranges marker_ranges = collapse_ranges(inline_diff_positions) + # Mark each range marker_ranges.each do |range| offset = insert_around_range(rich_line, range, "", "", offset) end @@ -22,6 +25,9 @@ module Gitlab rich_line end + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin mapping = [] @@ -31,6 +37,8 @@ module Gitlab raw_char = raw_line[raw_pos] rich_char = rich_line[rich_pos] + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment while rich_char == '<' until rich_char == '>' rich_pos += 1 @@ -50,6 +58,7 @@ module Gitlab end end + # Takes an array of integers, and returns an array of ranges covering the same integers def collapse_ranges(positions) return [] if positions.empty? ranges = [] @@ -71,6 +80,7 @@ module Gitlab ranges end + # Inserts tags around the characters identified by the given range def insert_around_range(text, range, before, after, offset = 0) text.insert(offset + range.begin, before) offset += before.length -- cgit v1.2.1 From e8785ded1e922a72989614d7e70e9c26c367f1aa Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 15 Jan 2016 17:47:28 -0500 Subject: Prevent StateMachine warnings from outputting during a cron task [ci skip] Closes #5931 --- lib/tasks/gitlab/task_helpers.rake | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index 8c63877e51c..d33b5b31e18 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -4,6 +4,9 @@ end String.disable_colorization = true unless STDOUT.isatty +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From bddef293d666c4b3933b93b017ae49bdf18b234f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 18 Jan 2016 22:43:04 -0200 Subject: Ensure that doesn't have pending migrations when running the specs --- lib/gitlab/current_settings.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7f938780ab1..ea054255820 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -39,7 +39,6 @@ module Gitlab end use_db && ActiveRecord::Base.connection.active? && - !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError -- cgit v1.2.1 From 0a8039eb7790426880bdd7b9d67775aeb6e5dac7 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 13:56:36 +0100 Subject: Remove useless assignments --- lib/gitlab/diff/inline_diff_marker.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 405465a641f..8998ccba5ce 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -31,10 +31,8 @@ module Gitlab def position_mapping @position_mapping ||= begin mapping = [] - raw_pos = 0 rich_pos = 0 (0..raw_line.length).each do |raw_pos| - raw_char = raw_line[raw_pos] rich_char = rich_line[rich_pos] # The raw and rich lines are the same except for HTML tags, -- cgit v1.2.1 From 512bebe21d7f57b691a1c8355581feb64b9b6292 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 14:52:41 +0100 Subject: Refactor Gitlab::Highlight and fix tests --- lib/gitlab/highlight.rb | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index a5b041687e3..28cfebef968 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,10 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, continue: false) - formatter = rouge_formatter(nowrap: nowrap) - - lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText - formatter.format(lexer.lex(blob_content, continue: continue)).html_safe + def self.highlight(blob_name, blob_content, nowrap: true) + new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) end def self.highlight_lines(repository, ref, file_name) @@ -14,9 +11,26 @@ module Gitlab highlight(file_name, blob.data).lines.map!(&:html_safe) end + def initialize(blob_name, blob_content, nowrap: true) + @formatter = rouge_formatter(nowrap: nowrap) + @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + end + + def highlight(text, continue: true) + @formatter.format(lex(text, continue: continue)).html_safe + end + private - def self.rouge_formatter(options = {}) + def lex(text, continue: true) + if @lexer == Rouge::Lexers::PlainText + @lexer.lex(text) + else + @lexer.lex(text, continue: continue) + end + end + + def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, cssclass: 'code highlight', -- cgit v1.2.1 From 5c7259c7c30228bf85a4efbbddb673046e8d733f Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 15:13:37 +0100 Subject: Don't crash when file can't be highlighted --- lib/gitlab/highlight.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 28cfebef968..4ddb4fea977 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -17,19 +17,13 @@ module Gitlab end def highlight(text, continue: true) - @formatter.format(lex(text, continue: continue)).html_safe + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + rescue + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end private - def lex(text, continue: true) - if @lexer == Rouge::Lexers::PlainText - @lexer.lex(text) - else - @lexer.lex(text, continue: continue) - end - end - def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, -- cgit v1.2.1 From 10669097fa7621a175755d144c61ecb80a5ea9fb Mon Sep 17 00:00:00 2001 From: Josh Frye Date: Wed, 13 Jan 2016 09:07:21 -0500 Subject: Add public params to GET /projects api. Closes #3788 --- lib/api/helpers.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6d2380cf47d..68d0aa671b6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,6 +264,10 @@ module API projects = projects.search(params[:search]) end + if params[:public].present? && parse_boolean(params[:public]) + projects = projects.public_only + end + projects.reorder(project_order_by => project_sort) end -- cgit v1.2.1 From 11797df1af483156b8cf11290c49c3f4d6089d99 Mon Sep 17 00:00:00 2001 From: Josh Frye Date: Mon, 18 Jan 2016 12:41:49 -0500 Subject: Change 'public' flag to more robust 'visibility' query. --- lib/api/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 68d0aa671b6..3f528b9f7c0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,8 +264,8 @@ module API projects = projects.search(params[:search]) end - if params[:public].present? && parse_boolean(params[:public]) - projects = projects.public_only + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) end projects.reorder(project_order_by => project_sort) -- cgit v1.2.1 From 98e1a5b63424b6912de98ee5055d3f9e57e63899 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 16:25:38 +0100 Subject: Allow LDAP users to change their email if it was not set by the LDAP server --- lib/gitlab/ldap/user.rb | 29 ++++++++++++++++------------- lib/gitlab/o_auth/auth_hash.rb | 8 ++++++-- lib/gitlab/o_auth/user.rb | 14 +++++++------- 3 files changed, 29 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index aef08c97d1d..e044f0ecc6d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -30,28 +30,31 @@ module Gitlab end def find_by_uid_and_provider - self.class.find_by_uid_and_provider( - auth_hash.uid, auth_hash.provider) + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? end def update_user_attributes - return unless persisted? + if persisted? + if auth_hash.has_email? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) + # For a new identity set extern_uid to the LDAP DN + # For an existing identity with matching email but changed DN, update the DN. + # For an existing identity with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid + end - # For a new user set extern_uid to the LDAP DN - # For an existing user with matching email but changed DN, update the DN. - # For an existing user with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid + gl_user.ldap_email = auth_hash.has_email? gl_user end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index ba31599432b..36e5c2670bb 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,6 +32,10 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end + def has_email? + get_info(:email).present? + end + private def info @@ -46,8 +50,8 @@ module Gitlab def username_and_email @username_and_email ||= begin - username = get_info(:username) || get_info(:nickname) - email = get_info(:email) + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence username ||= generate_username(email) if email email ||= generate_temporarily_email(username) if username diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e3d2cc65a8f..d87a72f7ba3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -111,7 +111,7 @@ module Gitlab def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users - else + else Gitlab.config.omniauth.block_auto_created_users end end @@ -135,16 +135,16 @@ module Gitlab def user_attributes # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? - username = ldap_person.username - email = ldap_person.email.first - else - username = auth_hash.username - email = auth_hash.email + username = ldap_person.username.presence + email = ldap_person.email.first.presence end + username ||= auth_hash.username + email ||= auth_hash.email + name = auth_hash.name name = ::Namespace.clean_path(username) if name.strip.empty? - + { name: name, username: ::Namespace.clean_path(username), -- cgit v1.2.1 From 84124380e815e549f50d82051b2b546c12d9a724 Mon Sep 17 00:00:00 2001 From: Anton Baklanov Date: Sat, 19 Dec 2015 21:04:40 +0200 Subject: Added X-GitLab-... headers to emails from CI and Email On Push services Fixes #2098 --- lib/gitlab/email/message/repository_push.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index a2eb7a70bd2..a05ffeb9cd2 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -9,6 +9,7 @@ module Gitlab delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author + delegate :username, to: :author, prefix: :author def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless -- cgit v1.2.1 From 701513dcc7afb403372bc738642a9a52e9be5983 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 14:51:56 +0100 Subject: Move parallel diff logic to separate class --- lib/gitlab/diff/file.rb | 4 ++ lib/gitlab/diff/parallel_diff.rb | 117 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 lib/gitlab/diff/parallel_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index a6a7fc8ff4c..74b1c117129 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -20,6 +20,10 @@ module Gitlab Gitlab::Diff::Highlight.new(self).highlight end + def parallel_diff_lines + Gitlab::Diff::ParallelDiff.new(self).parallelize + end + def mode_changed? !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb new file mode 100644 index 00000000000..a0dc3da875d --- /dev/null +++ b/lib/gitlab/diff/parallel_diff.rb @@ -0,0 +1,117 @@ +module Gitlab + module Diff + class ParallelDiff + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + def parallelize + lines = [] + skip_next = false + + diff_file.highlighted_diff_lines.each do |line| + full_line = line.text + type = line.type + line_code = generate_line_code(diff_file.file_path, line) + line_new = line.new_pos + line_old = line.old_pos + + next_line = diff_file.next_line(line.index) + + if next_line + next_line_code = generate_line_code(diff_file.file_path, next_line) + next_type = next_line.type + next_line = next_line.text + end + + case type + when 'match', nil + # line in the right panel is the same as in the left one + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + when 'old' + case next_type + when 'new' + # Left side has text removed, right side has text added + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: line_new, + text: next_line, + line_code: next_line_code + } + } + skip_next = true + when 'old', nil + # Left side has text removed, right side doesn't have any change + # No next line code, no new line number, no new line text + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: nil, + text: "", + line_code: nil + } + } + end + when 'new' + if skip_next + # Change has been already included in previous line so no need to do it again + skip_next = false + next + else + # Change is only on the right side, left side has no change + lines << { + left: { + type: nil, + number: nil, + text: "", + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + end + end + end + lines + end + + private + + def generate_line_code(file_path, line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + end + end +end -- cgit v1.2.1 From a010db5db243a532cb8d1c2d5ac787e90da0044f Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 15:26:44 +0100 Subject: Properly handle HTML entities with inline diffs --- lib/gitlab/diff/inline_diff.rb | 11 ++++------- lib/gitlab/diff/inline_diff_marker.rb | 16 ++++++++++++++-- lib/gitlab/diff/parser.rb | 7 +------ 3 files changed, 19 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index e5986fd69e2..b8a61ad6115 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -16,15 +16,12 @@ module Gitlab old_line = @lines[old_index] new_line = @lines[new_index] - suffixless_old_line = old_line[1..-1] - suffixless_new_line = new_line[1..-1] - # Skip inline diff if empty line was replaced with content - next if suffixless_old_line == "" + next if old_line[1..-1] == "" - # Add one, because this is based on the suffixless version - lcp = longest_common_prefix(suffixless_old_line, suffixless_new_line) + 1 - lcs = longest_common_suffix(suffixless_old_line, suffixless_new_line) + # Add one, because this is based on the prefixless version + lcp = longest_common_prefix(old_line[1..-1], new_line[1..-1]) + 1 + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) old_diff_range = lcp..(old_line.length - lcs - 1) new_diff_range = lcp..(new_line.length - lcs - 1) diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 8998ccba5ce..c31149d374e 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -12,7 +12,7 @@ module Gitlab offset = 0 line_inline_diffs.each do |inline_diff_range| # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range] + inline_diff_positions = position_mapping[inline_diff_range].flatten # Turn the array of character positions into ranges marker_ranges = collapse_ranges(inline_diff_positions) @@ -47,7 +47,19 @@ module Gitlab rich_char = rich_line[rich_pos] end - mapping[raw_pos] = rich_pos + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end rich_pos += 1 end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 2c0a866e8ba..6c8a1fc6d6f 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -14,7 +14,7 @@ module Gitlab @lines.each do |line| next if filename?(line) - full_line = html_escape(line.gsub(/\n/, '')) + full_line = line.gsub(/\n/, '') if line.match(/^@@ -/) type = "match" @@ -67,11 +67,6 @@ module Gitlab nil end end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end -- cgit v1.2.1 From 8536e083f7b2d7ed77ecae83774d75f68d66e0b4 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 20 Jan 2016 13:16:34 +0100 Subject: Add IP blocking against DNSBL at sign-up --- lib/dnsxl_check.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/ip_check.rb | 34 ++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 lib/dnsxl_check.rb create mode 100644 lib/gitlab/ip_check.rb (limited to 'lib') diff --git a/lib/dnsxl_check.rb b/lib/dnsxl_check.rb new file mode 100644 index 00000000000..1e506b2d9cb --- /dev/null +++ b/lib/dnsxl_check.rb @@ -0,0 +1,105 @@ +require 'resolv' + +class DNSXLCheck + + class Resolver + def self.search(query) + begin + Resolv.getaddress(query) + true + rescue Resolv::ResolvError + false + end + end + end + + IP_REGEXP = /\A(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\z/ + DEFAULT_THRESHOLD = 0.33 + + def self.create_from_list(list) + dnsxl_check = DNSXLCheck.new + + list.each do |entry| + dnsxl_check.add_list(entry.domain, entry.weight) + end + + dnsxl_check + end + + def test(ip) + if use_threshold? + test_with_threshold(ip) + else + test_strict(ip) + end + end + + def test_with_threshold(ip) + return false if lists.empty? + + search(ip) + final_score >= threshold + end + + def test_strict(ip) + return false if lists.empty? + + search(ip) + @score > 0 + end + + def use_threshold=(value) + @use_threshold = value == true + end + + def use_threshold? + @use_threshold &&= true + end + + def threshold=(threshold) + raise ArgumentError, "'threshold' value must be grather than 0 and less than or equal to 1" unless threshold > 0 && threshold <= 1 + @threshold = threshold + end + + def threshold + @threshold ||= DEFAULT_THRESHOLD + end + + def add_list(domain, weight) + @lists ||= [] + @lists << { domain: domain, weight: weight } + end + + def lists + @lists ||= [] + end + + private + + def search(ip) + raise ArgumentError, "'ip' value must be in #{IP_REGEXP} format" unless ip.match(IP_REGEXP) + + @score = 0 + + reversed = reverse_ip(ip) + search_in_rbls(reversed) + end + + def reverse_ip(ip) + ip.split('.').reverse.join('.') + end + + def search_in_rbls(reversed_ip) + lists.each do |rbl| + query = "#{reversed_ip}.#{rbl[:domain]}" + @score += rbl[:weight] if Resolver.search(query) + end + end + + def final_score + weights = lists.map{ |rbl| rbl[:weight] }.reduce(:+).to_i + return 0 if weights == 0 + + (@score.to_f / weights.to_f).round(2) + end +end diff --git a/lib/gitlab/ip_check.rb b/lib/gitlab/ip_check.rb new file mode 100644 index 00000000000..f2e9b50d225 --- /dev/null +++ b/lib/gitlab/ip_check.rb @@ -0,0 +1,34 @@ +module Gitlab + class IpCheck + + def initialize(ip) + @ip = ip + + application_settings = ApplicationSetting.current + @ip_blocking_enabled = application_settings.ip_blocking_enabled + @dnsbl_servers_list = application_settings.dnsbl_servers_list + end + + def spam? + @ip_blocking_enabled && blacklisted? + end + + private + + def blacklisted? + on_dns_blacklist? + end + + def on_dns_blacklist? + dnsbl_check = DNSXLCheck.new + prepare_dnsbl_list(dnsbl_check) + dnsbl_check.test(@ip) + end + + def prepare_dnsbl_list(dnsbl_check) + @dnsbl_servers_list.split(',').map(&:strip).reject(&:empty?).each do |domain| + dnsbl_check.add_list(domain, 1) + end + end + end +end -- cgit v1.2.1 From a10ab94b068c31601c7d4ab0062b9d567af6cee2 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 13 Jan 2016 07:05:28 -0800 Subject: Gracefully handle invalid UTF-8 sequences in Markdown links Closes #6077 --- lib/banzai/filter/reference_filter.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 20bd4f7ee6e..3637b1bac94 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,6 +133,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,6 +171,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 577f2fb47a7ef57415b78b49d5c746d4e99f6a98 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 18:44:27 +0100 Subject: Save and use actual diff base commit for MR diff highlighting --- lib/gitlab/diff/file.rb | 12 ++++++++++-- lib/gitlab/diff/highlight.rb | 9 ++++++++- lib/gitlab/diff/inline_diff_marker.rb | 13 +++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 74b1c117129..a484177ae33 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,14 +1,22 @@ module Gitlab module Diff class File - attr_reader :diff, :new_ref, :old_ref + attr_reader :diff, :diff_refs delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false def initialize(diff, diff_refs) @diff = diff - @old_ref, @new_ref = diff_refs + @diff_refs = diff_refs + end + + def old_ref + diff_refs[0] if diff_refs + end + + def new_ref + diff_refs[1] if diff_refs end # Array of Gitlab::DIff::Line objects diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b6875f07279..fe084a7399a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -31,6 +31,8 @@ module Gitlab private def highlight_line(diff_line, index) + return html_escape(diff_line.text) unless diff_file.diff_refs + line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' case diff_line.type @@ -42,7 +44,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - rich_line ? line_prefix + rich_line : diff_line.text + rich_line ? line_prefix + rich_line : html_escape(diff_line.text) end def inline_diffs @@ -63,6 +65,11 @@ module Gitlab [ref.project.repository, ref.id, path] end + + def html_escape(str) + replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } + str.gsub(/[&"'><]/, replacements) + end end end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index c31149d374e..7ca198eeb86 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -9,17 +9,18 @@ module Gitlab end def mark(line_inline_diffs) - offset = 0 + marker_ranges = [] line_inline_diffs.each do |inline_diff_range| # Map the inline-diff range based on the raw line to character positions in the rich line inline_diff_positions = position_mapping[inline_diff_range].flatten # Turn the array of character positions into ranges - marker_ranges = collapse_ranges(inline_diff_positions) + marker_ranges.concat(collapse_ranges(inline_diff_positions)) + end - # Mark each range - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end + offset = 0 + # Mark each range + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) end rich_line -- cgit v1.2.1 From 0e992a3b4e710f8486a37bfa73ad6981365fceb2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 19:20:13 +0100 Subject: Properly highlight lines around '\ No newline at end of file' --- lib/gitlab/diff/highlight.rb | 4 ++-- lib/gitlab/diff/parallel_diff.rb | 2 +- lib/gitlab/diff/parser.rb | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index fe084a7399a..179f8164c84 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,7 +14,7 @@ module Gitlab def highlight @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines - next if diff_line.type == 'match' + next if diff_line.type == 'match' || diff_line.type == 'nonewline' rich_line = highlight_line(diff_line, i) @@ -33,7 +33,7 @@ module Gitlab def highlight_line(diff_line, index) return html_escape(diff_line.text) unless diff_file.diff_refs - line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' case diff_line.type when 'new', nil diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index a0dc3da875d..c0db3559e3a 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -62,7 +62,7 @@ module Gitlab } } skip_next = true - when 'old', nil + when 'old', 'nonewline', nil # Left side has text removed, right side doesn't have any change # No next line code, no new line number, no new line text lines << { diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 6c8a1fc6d6f..3666063bf8b 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -26,6 +26,10 @@ module Gitlab lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next + elsif line[0] == '\\' + type = 'nonewline' + lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 else type = identification_type(line) lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) @@ -33,10 +37,13 @@ module Gitlab end - if line[0] == "+" + case line[0] + when "+" line_new += 1 - elsif line[0] == "-" + when "-" line_old += 1 + when "\\" + # No increment else line_new += 1 line_old += 1 @@ -59,9 +66,10 @@ module Gitlab end def identification_type(line) - if line[0] == "+" + case line[0] + when "+" "new" - elsif line[0] == "-" + when "-" "old" else nil -- cgit v1.2.1 From 6000f8545f43b449035cb50382901ce40fb807b0 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 19:33:34 +0100 Subject: Validate bounds just to be sure --- lib/gitlab/diff/inline_diff_marker.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 7ca198eeb86..1d7fa1bce06 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -93,6 +93,9 @@ module Gitlab # Inserts tags around the characters identified by the given range def insert_around_range(text, range, before, after, offset = 0) + # Just to be sure + return offset if offset + range.end + 1 > text.length + text.insert(offset + range.begin, before) offset += before.length -- cgit v1.2.1 From f4bdbecfd54c39baed79d0143de97933d26e3215 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 20 Jan 2016 19:36:35 -0200 Subject: Fix cross projects detection when importing GitHub pull requests --- lib/gitlab/github_import/pull_request_formatter.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b7c47958cc7..f96fed0f5cf 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -18,7 +18,7 @@ module Gitlab end def cross_project? - source_repo.fork == true + source_repo.id != target_repo.id end def number @@ -73,6 +73,10 @@ module Gitlab project end + def target_repo + raw_data.base.repo + end + def target_branch target_project.repository.find_branch(raw_data.base.ref) end -- cgit v1.2.1 From c45a6bf3ba13cbd532852dfcc48ef3fd7aa545e4 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 14 Jan 2016 17:52:59 +0100 Subject: Added cache:key to .gitlab-ci.yml allowing to fine tune the caching --- lib/ci/gitlab_ci_yaml_processor.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index bcdfd38d292..1a3f662811a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -115,6 +115,10 @@ module Ci end if @cache + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" + end + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) raise ValidationError, "cache:untracked parameter should be an boolean" end @@ -198,6 +202,10 @@ module Ci end def validate_job_cache!(name, job) + if job[:cache][:key] && !validate_string(job[:cache][:key]) + raise ValidationError, "#{name} job: cache:key parameter should be a string" + end + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" end -- cgit v1.2.1 From 693fa7a458645291cc903bf400f9b463941cb63d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 21 Jan 2016 13:36:55 +0100 Subject: Init script changes for gitlab-workhorse in 8.4 --- lib/support/init.d/gitlab | 9 ++++++--- lib/support/init.d/gitlab.default.example | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f07c8b508..1633891c8a0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -38,6 +38,7 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" @@ -233,10 +234,12 @@ start_gitlab() { if [ "$gitlab_workhorse_status" = "0" ]; then echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-workhorse does this itself + # No need to remove a socket, gitlab-workhorse does this itself. + # Because gitlab-workhorse has multiple executables we need to fix + # the PATH. $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ - $app_root/../gitlab-workhorse/gitlab-workhorse \ - $gitlab_workhorse_options \ + /usr/bin/env PATH=$gitlab_workhorse_dir:$PATH \ + gitlab-workhorse $gitlab_workhorse_options \ >> $gitlab_workhorse_log 2>&1 & fi diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1937ca582b0..4e6e56ac2db 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,6 +30,9 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# The directory where the gitlab-workhorse binaries are. Usually +# /home/git/gitlab-workhorse . +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write -- cgit v1.2.1 From 4861b1bbfbd962416452e1e883c67b885678b5a0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Jan 2016 13:11:38 -0200 Subject: Fix import of GitHub's wiki when the repository has not been created --- lib/gitlab/github_import/importer.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 18929b9113b..663402e8197 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -82,8 +82,12 @@ module Gitlab end true - rescue Gitlab::Shell::Error - false + rescue Gitlab::Shell::Error => e + if e.message =~ /repository not exported/ + true + else + false + end end end end -- cgit v1.2.1