From 34657b821ae597de76ffd5a70d2b0b298dc270ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 15 Dec 2015 18:09:09 -0500 Subject: Add syntax highlighting to diff view. #3945 --- lib/rouge/lexers/gitlab_diff.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 lib/rouge/lexers/gitlab_diff.rb (limited to 'lib') diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb new file mode 100644 index 00000000000..e136d47df00 --- /dev/null +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -0,0 +1,20 @@ +Rouge::Token::Tokens.token(:InlineDiff, 'idiff') + +module Rouge + module Lexers + class GitlabDiff < RegexLexer + title "GitLab Diff" + tag 'gitlab_diff' + + state :root do + rule %r{(.*?)} do |match| + token InlineDiff, match[1] + end + + rule /(?:(?! Date: Wed, 23 Dec 2015 19:10:13 -0500 Subject: Set initial state on parent Lexer. #3945 --- lib/rouge/lexers/gitlab_diff.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index e136d47df00..c7aaeb92608 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -15,6 +15,10 @@ module Rouge delegate option(:parent_lexer) end end + + start do + option(:parent_lexer).reset! + end end end end -- cgit v1.2.1 From bb96d631537d3d8181f0d3b762603a012219c3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 00:52:50 -0500 Subject: New implementation for highlighting diff files. #3945 * It is more performant given now we process all the diff file instead of processing line by line. * Multiline comments are highlighted correctly. --- lib/gitlab/diff/file.rb | 4 +++ lib/gitlab/diff/highlight.rb | 55 +++++++++++++++++++++++++++++++++++++++++ lib/gitlab/diff/line.rb | 1 + lib/rouge/lexers/gitlab_diff.rb | 2 +- 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/diff/highlight.rb (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 79061cd0141..ff8765b8e26 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -15,6 +15,10 @@ module Gitlab @lines ||= parser.parse(raw_diff.lines) end + def highlighted_diff_lines + Gitlab::Diff::Highlight.process_diff_lines(self) + end + def mode_changed? !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb new file mode 100644 index 00000000000..f10b55eb00b --- /dev/null +++ b/lib/gitlab/diff/highlight.rb @@ -0,0 +1,55 @@ +module Gitlab + module Diff + class Highlight + def self.process_diff_lines(diff_file) + processor = new(diff_file) + processor.highlight + end + + def initialize(diff_file) + text_lines = diff_file.diff_lines.map(&:text) + @diff_file = diff_file + @diff_lines = diff_file.diff_lines + @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + @raw_lines = text_lines.join("\n") + end + + def highlight + @code = unescape_html(@raw_lines) + @highlighted_code = formatter.format(lexer.lex(@code)) + + update_diff_lines + end + + private + + def update_diff_lines + @highlighted_code.lines.each_with_index do |line, i| + @diff_lines[i].highlighted_text = "#{@diff_line_prefixes[i]}#{line}" + end + + @diff_lines + end + + def lexer + parent = Rouge::Lexer.guess(filename: @diff_file.new_path, source: @code).new rescue Rouge::Lexers::PlainText.new + Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) + end + + def unescape_html(content) + text = CGI.unescapeHTML(content) + text.gsub!(' ', ' ') + text + end + + def formatter + @formatter ||= Rouge::Formatters::HTMLGitlab.new( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + end + end + end +end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0072194606e..c48c69fb344 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,6 +2,7 @@ module Gitlab module Diff class Line attr_reader :type, :text, :index, :old_pos, :new_pos + attr_accessor :highlighted_text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index c7aaeb92608..d91dd6c4245 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -11,7 +11,7 @@ module Rouge token InlineDiff, match[1] end - rule /(?:(?! Date: Wed, 30 Dec 2015 13:10:28 -0500 Subject: Fix Rubocop complain. #3945 --- lib/gitlab/diff/highlight.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f10b55eb00b..adb437abed2 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -44,11 +44,11 @@ module Gitlab def formatter @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) end end end -- cgit v1.2.1 From 7de90f4b53f865dc417d022a9133372e57274549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 18:42:11 -0500 Subject: Fix broken spec and small refactor. #3945 --- lib/gitlab/diff/highlight.rb | 2 +- lib/gitlab/diff/line.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index adb437abed2..d0c2e3670c6 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -25,7 +25,7 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - @diff_lines[i].highlighted_text = "#{@diff_line_prefixes[i]}#{line}" + @diff_lines[i].text = "#{@diff_line_prefixes[i]}#{line}" end @diff_lines diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index c48c69fb344..03730b435ad 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,8 +1,8 @@ module Gitlab module Diff class Line - attr_reader :type, :text, :index, :old_pos, :new_pos - attr_accessor :highlighted_text + attr_reader :type, :index, :old_pos, :new_pos + attr_accessor :text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index -- cgit v1.2.1 From 8b079315d98a8ccf852592148632c6f052d9cb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 21:23:50 -0500 Subject: A bit of refactoring. #3945 --- lib/gitlab/diff/file.rb | 2 +- lib/gitlab/diff/highlight.rb | 16 ++++++++-------- lib/rouge/lexers/gitlab_diff.rb | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ff8765b8e26..69b38a32eeb 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -16,7 +16,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(self) + Gitlab::Diff::Highlight.process_diff_lines(new_path, diff_lines) end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index d0c2e3670c6..40a54ede2bb 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,15 +1,15 @@ module Gitlab module Diff class Highlight - def self.process_diff_lines(diff_file) - processor = new(diff_file) + def self.process_diff_lines(file_name, diff_lines) + processor = new(file_name, diff_lines) processor.highlight end - def initialize(diff_file) - text_lines = diff_file.diff_lines.map(&:text) - @diff_file = diff_file - @diff_lines = diff_file.diff_lines + def initialize(file_name, diff_lines) + text_lines = diff_lines.map(&:text) + @file_name = file_name + @diff_lines = diff_lines @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } @raw_lines = text_lines.join("\n") end @@ -32,7 +32,7 @@ module Gitlab end def lexer - parent = Rouge::Lexer.guess(filename: @diff_file.new_path, source: @code).new rescue Rouge::Lexers::PlainText.new + parent = Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) end @@ -43,7 +43,7 @@ module Gitlab end def formatter - @formatter ||= Rouge::Formatters::HTMLGitlab.new( + Rouge::Formatters::HTMLGitlab.new( nowrap: true, cssclass: 'code highlight', lineanchors: true, diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb index d91dd6c4245..cbf272ee1de 100644 --- a/lib/rouge/lexers/gitlab_diff.rb +++ b/lib/rouge/lexers/gitlab_diff.rb @@ -2,6 +2,8 @@ Rouge::Token::Tokens.token(:InlineDiff, 'idiff') module Rouge module Lexers + # This new Lexer is required in order to avoid the inline diff markup + # to be tokenized, it will be rendered as raw HTML code if that happens. class GitlabDiff < RegexLexer title "GitLab Diff" tag 'gitlab_diff' -- cgit v1.2.1 From fd100e1ef1726418c81ab8833cf8bcf86fab6eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 30 Dec 2015 21:44:12 -0500 Subject: Don't modify "match" diff lines. #3945 --- lib/gitlab/diff/highlight.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 40a54ede2bb..c780ea21775 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -25,7 +25,12 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - @diff_lines[i].text = "#{@diff_line_prefixes[i]}#{line}" + diff_line = @diff_lines[i] + + # ignore highlighting for "match" lines + next if diff_line.type == 'match' + + diff_line.text = "#{@diff_line_prefixes[i]}#{line}" end @diff_lines -- cgit v1.2.1 From 3fbcf52ec8decc3a4e331d52b2f47d7b85d399cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 31 Dec 2015 01:05:52 -0500 Subject: Apply syntax highlighting when expanding diff plus some refactor. #3945 --- lib/gitlab/diff/highlight.rb | 55 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index c780ea21775..7f340de65cc 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,31 +1,62 @@ module Gitlab module Diff class Highlight - def self.process_diff_lines(file_name, diff_lines) - processor = new(file_name, diff_lines) + # Apply syntax highlight to provided source code + # + # file_name - The file name related to the code. + # lines - It can be an Array of Gitlab::Diff::Line objects or simple Strings. + # When passing Strings you need to provide the required 'end of lines' + # chars ("\n") for each String given that we don't append them automatically. + # + # Returns an Array with the processed items. + def self.process_diff_lines(file_name, lines) + processor = new(file_name, lines) processor.highlight end - def initialize(file_name, diff_lines) - text_lines = diff_lines.map(&:text) - @file_name = file_name - @diff_lines = diff_lines - @diff_line_prefixes = text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } - @raw_lines = text_lines.join("\n") + def initialize(file_name, lines) + @file_name = file_name + @lines = lines end def highlight - @code = unescape_html(@raw_lines) + return [] if @lines.empty? + + extract_line_prefixes + + @code = unescape_html(raw_content) @highlighted_code = formatter.format(lexer.lex(@code)) - update_diff_lines + is_diff_line? ? update_diff_lines : @highlighted_code.lines end private + def is_diff_line? + @lines.first.is_a?(Gitlab::Diff::Line) + end + + def text_lines + @text_lines ||= (is_diff_line? ? @lines.map(&:text) : @lines) + end + + def raw_content + @raw_content ||= text_lines.join(is_diff_line? ? "\n" : nil) + end + + def extract_line_prefixes + @diff_line_prefixes ||= begin + if is_diff_line? + text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + else + [] + end + end + end + def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| - diff_line = @diff_lines[i] + diff_line = @lines[i] # ignore highlighting for "match" lines next if diff_line.type == 'match' @@ -33,7 +64,7 @@ module Gitlab diff_line.text = "#{@diff_line_prefixes[i]}#{line}" end - @diff_lines + @lines end def lexer -- cgit v1.2.1 From f1f4fdf778245cab74ff9cda2a421315c21a99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 7 Jan 2016 21:08:57 -0500 Subject: Don't process inline diffs on backend. #3945 --- lib/gitlab/diff/parser.rb | 5 +-- lib/gitlab/inline_diff.rb | 104 ---------------------------------------------- 2 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 lib/gitlab/inline_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..177bad4b1cf 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -11,13 +11,10 @@ module Gitlab line_new = 1 type = nil - lines_arr = ::Gitlab::InlineDiff.processing lines - - lines_arr.each do |line| + @lines.each do |line| next if filename?(line) full_line = html_escape(line.gsub(/\n/, '')) - full_line = ::Gitlab::InlineDiff.replace_markers full_line if line.match(/^@@ -/) type = "match" diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb deleted file mode 100644 index 44507bde25d..00000000000 --- a/lib/gitlab/inline_diff.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Gitlab - class InlineDiff - class << self - - START = "#!idiff-start!#" - FINISH = "#!idiff-finish!#" - - def processing(diff_arr) - indexes = _indexes_of_changed_lines diff_arr - - indexes.each do |index| - first_line = diff_arr[index+1] - second_line = diff_arr[index+2] - - # Skip inline diff if empty line was replaced with content - next if first_line == "-\n" - - first_token = find_first_token(first_line, second_line) - apply_first_token(diff_arr, index, first_token) - - last_token = find_last_token(first_line, second_line, first_token) - apply_last_token(diff_arr, index, last_token) - end - - diff_arr - end - - def apply_first_token(diff_arr, index, first_token) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end - end - - def apply_last_token(diff_arr, index, last_token) - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - end - - def find_first_token(first_line, second_line) - max_length = [first_line.size, second_line.size].max - first_the_same_symbols = 0 - - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_line[0..first_the_same_symbols][1..-1] - end - - def find_last_token(first_line, second_line, first_token) - max_length = [first_line.size, second_line.size].max - last_the_same_symbols = 0 - - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - - if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - - last_the_same_symbols += 1 - first_line[last_the_same_symbols..-1] - end - - def _indexes_of_changed_lines(diff_arr) - chain_of_first_symbols = "" - diff_arr.each_with_index do |line, i| - chain_of_first_symbols += line[0] - end - chain_of_first_symbols.gsub!(/[^\-\+]/, "#") - - offset = 0 - indexes = [] - while index = chain_of_first_symbols.index("#-+#", offset) - indexes << index - offset = index + 1 - end - indexes - end - - def replace_markers(line) - line.gsub!(START, "") - line.gsub!(FINISH, "") - line - end - end - end -end -- cgit v1.2.1 From 21b602c60ad787b63039d804a5e15b43d0d3c32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 7 Jan 2016 22:37:01 -0500 Subject: Change strategy to highlight diffs. #3945 Now we apply syntax highlighting to the whole old and new files. This basically help us to highlight adequately multiline content. --- lib/gitlab/diff/file.rb | 8 +++-- lib/gitlab/diff/highlight.rb | 70 +++++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 69b38a32eeb..cb93c6a574a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,13 +1,15 @@ module Gitlab module Diff class File - attr_reader :diff + attr_reader :diff, :repository, :new_ref, :old_ref delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff) + def initialize(diff, diff_refs, repository) @diff = diff + @repository = repository + @old_ref, @new_ref = diff_refs end # Array of Gitlab::DIff::Line objects @@ -16,7 +18,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(new_path, diff_lines) + Gitlab::Diff::Highlight.process_diff_lines(self) end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7f340de65cc..0d0a3268107 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,22 +1,43 @@ module Gitlab module Diff class Highlight + attr_reader :diff_file + + delegate :repository, :old_path, :new_path, :old_ref, :new_ref, + to: :diff_file, prefix: :diff + # Apply syntax highlight to provided source code # - # file_name - The file name related to the code. - # lines - It can be an Array of Gitlab::Diff::Line objects or simple Strings. - # When passing Strings you need to provide the required 'end of lines' - # chars ("\n") for each String given that we don't append them automatically. + # diff_file - an instance of Gitlab::Diff::File # # Returns an Array with the processed items. - def self.process_diff_lines(file_name, lines) - processor = new(file_name, lines) + def self.process_diff_lines(diff_file) + processor = new(diff_file) processor.highlight end - def initialize(file_name, lines) - @file_name = file_name - @lines = lines + def self.process_file(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + content = blob.data + lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new + formatter.format(lexer.lex(content)).lines + end + + def self.formatter + @formatter ||= Rouge::Formatters::HTMLGitlab.new( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + end + + def initialize(diff_file) + @diff_file = diff_file + @file_name = diff_file.new_path + @lines = diff_file.diff_lines end def highlight @@ -47,7 +68,7 @@ module Gitlab def extract_line_prefixes @diff_line_prefixes ||= begin if is_diff_line? - text_lines.map { |line| line.sub!(/\A((\+|\-)\s*)/, '');$1 } + text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } else [] end @@ -57,11 +78,17 @@ module Gitlab def update_diff_lines @highlighted_code.lines.each_with_index do |line, i| diff_line = @lines[i] + line_prefix = @diff_line_prefixes[i] || ' ' # ignore highlighting for "match" lines next if diff_line.type == 'match' - diff_line.text = "#{@diff_line_prefixes[i]}#{line}" + case diff_line.type + when 'new', nil + diff_line.text = new_lines[diff_line.new_pos - 1].try(:gsub!, /\A\s/, line_prefix) + when 'old' + diff_line.text = old_lines[diff_line.old_pos - 1].try(:gsub!, /\A\s/, line_prefix) + end end @lines @@ -79,12 +106,21 @@ module Gitlab end def formatter - Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + self.class.formatter + end + + def old_lines + @old_lines ||= begin + lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) + lines.map! { |line| " #{line}" } + end + end + + def new_lines + @new_lines ||= begin + lines = self.class.process_file(diff_repository, diff_new_ref, diff_new_path) + lines.map! { |line| " #{line}" } + end end end end -- cgit v1.2.1 From 6282202ee84f80f2197698b6f132abdf588e94d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Fri, 8 Jan 2016 15:17:45 -0500 Subject: Remove custom Lexer. #3945 [ci skip] Inline diff is going to be generated client side now. #3945 --- lib/gitlab/diff/highlight.rb | 3 +-- lib/rouge/lexers/gitlab_diff.rb | 26 -------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 lib/rouge/lexers/gitlab_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0d0a3268107..7dd44b6004a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -95,8 +95,7 @@ module Gitlab end def lexer - parent = Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new - Rouge::Lexers::GitlabDiff.new(parent_lexer: parent) + Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new end def unescape_html(content) diff --git a/lib/rouge/lexers/gitlab_diff.rb b/lib/rouge/lexers/gitlab_diff.rb deleted file mode 100644 index cbf272ee1de..00000000000 --- a/lib/rouge/lexers/gitlab_diff.rb +++ /dev/null @@ -1,26 +0,0 @@ -Rouge::Token::Tokens.token(:InlineDiff, 'idiff') - -module Rouge - module Lexers - # This new Lexer is required in order to avoid the inline diff markup - # to be tokenized, it will be rendered as raw HTML code if that happens. - class GitlabDiff < RegexLexer - title "GitLab Diff" - tag 'gitlab_diff' - - state :root do - rule %r{(.*?)} do |match| - token InlineDiff, match[1] - end - - rule /(?:(?! Date: Fri, 8 Jan 2016 19:05:55 -0500 Subject: Fix broken specs. #3945 --- lib/gitlab/diff/highlight.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7dd44b6004a..caeefe5bbdd 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -27,11 +27,11 @@ module Gitlab def self.formatter @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) end def initialize(diff_file) -- cgit v1.2.1 From fed10766e533fcef2a6840deeb8d7ea1747f0c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Sat, 9 Jan 2016 01:55:31 -0500 Subject: Fix broken spec for submodule commit. #3945 --- lib/gitlab/diff/highlight.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index caeefe5bbdd..3c44abff3fb 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -85,10 +85,14 @@ module Gitlab case diff_line.type when 'new', nil - diff_line.text = new_lines[diff_line.new_pos - 1].try(:gsub!, /\A\s/, line_prefix) + line = new_lines[diff_line.new_pos - 1] when 'old' - diff_line.text = old_lines[diff_line.old_pos - 1].try(:gsub!, /\A\s/, line_prefix) + line = old_lines[diff_line.old_pos - 1] end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + diff_line.text = line.gsub!(/\A\s/, line_prefix) if line end @lines @@ -121,6 +125,10 @@ module Gitlab lines.map! { |line| " #{line}" } end end + + def submodules + @submodules ||= diff_repository.raw_repository.submodules(diff_new_ref).keys + end end end end -- cgit v1.2.1 From f1f9b5f7d388c6d7a0938229c9211beddb2fd6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 12:53:54 -0500 Subject: Small fixes from code review. #3945 --- lib/gitlab/diff/highlight.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 3c44abff3fb..d0137ab5f08 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -125,10 +125,6 @@ module Gitlab lines.map! { |line| " #{line}" } end end - - def submodules - @submodules ||= diff_repository.raw_repository.submodules(diff_new_ref).keys - end end end end -- cgit v1.2.1 From 6e3358a5077f5c6052e733722cd6baa63e43c081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 17:36:08 -0500 Subject: Remove no longer required code. #3945 --- lib/gitlab/diff/highlight.rb | 51 ++++++++------------------------------------ 1 file changed, 9 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index d0137ab5f08..14cf98e24a2 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -44,74 +44,41 @@ module Gitlab return [] if @lines.empty? extract_line_prefixes - - @code = unescape_html(raw_content) - @highlighted_code = formatter.format(lexer.lex(@code)) - - is_diff_line? ? update_diff_lines : @highlighted_code.lines + update_diff_lines end private - def is_diff_line? - @lines.first.is_a?(Gitlab::Diff::Line) - end - def text_lines - @text_lines ||= (is_diff_line? ? @lines.map(&:text) : @lines) - end - - def raw_content - @raw_content ||= text_lines.join(is_diff_line? ? "\n" : nil) + @text_lines ||= @lines.map(&:text) end def extract_line_prefixes - @diff_line_prefixes ||= begin - if is_diff_line? - text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } - else - [] - end - end + @diff_line_prefixes ||= text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } end def update_diff_lines - @highlighted_code.lines.each_with_index do |line, i| - diff_line = @lines[i] + @lines.each_with_index do |line, i| line_prefix = @diff_line_prefixes[i] || ' ' # ignore highlighting for "match" lines - next if diff_line.type == 'match' + next if line.type == 'match' - case diff_line.type + case line.type when 'new', nil - line = new_lines[diff_line.new_pos - 1] + highlighted_line = new_lines[line.new_pos - 1] when 'old' - line = old_lines[diff_line.old_pos - 1] + highlighted_line = old_lines[line.old_pos - 1] end # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - diff_line.text = line.gsub!(/\A\s/, line_prefix) if line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if line end @lines end - def lexer - Rouge::Lexer.guess(filename: @file_name, source: @code).new rescue Rouge::Lexers::PlainText.new - end - - def unescape_html(content) - text = CGI.unescapeHTML(content) - text.gsub!(' ', ' ') - text - end - - def formatter - self.class.formatter - end - def old_lines @old_lines ||= begin lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) -- cgit v1.2.1 From 7307fa48e70a9fb42902c8a84479d4cf669640aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Tue, 12 Jan 2016 19:28:31 -0500 Subject: Fix broken specs. #3945 --- lib/gitlab/diff/highlight.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 14cf98e24a2..0b6a348acbc 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if highlighted_line end @lines -- cgit v1.2.1 From 0f0af19139db71255934e9a7a5b5cd86420b7186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 13 Jan 2016 11:39:15 -0500 Subject: Little refactor for usage of html_safe. #3945 --- lib/gitlab/diff/highlight.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0b6a348acbc..f940b57d596 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -22,7 +22,7 @@ module Gitlab content = blob.data lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new - formatter.format(lexer.lex(content)).lines + formatter.format(lexer.lex(content)).lines.map!(&:html_safe) end def self.formatter @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix) if highlighted_line + line.text = highlighted_line.gsub!(/\A\s/, line_prefix).html_safe if highlighted_line end @lines -- cgit v1.2.1 From c179b48c97be22ef55ef9f5874984f7359fb12f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 13 Jan 2016 21:04:53 -0500 Subject: Use #sub instead of #gsub!. #3945 * This is because is not a good idea to modify the original lines * Also I run into this issue https://gitlab.com/gitlab-org/gitlab_git/issues/14 which is returning Diff Lines with the same @new_pos value. --- lib/gitlab/diff/highlight.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f940b57d596..e76a6f27856 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -73,7 +73,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.gsub!(/\A\s/, line_prefix).html_safe if highlighted_line + line.text = highlighted_line.sub(/\A\s/, line_prefix).html_safe if highlighted_line end @lines -- cgit v1.2.1 From 3a1d0535992594bc77320f081d1f20b760b1c1f7 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 15:11:50 +0100 Subject: Remove duplication around highlighting. --- lib/gitlab/diff/highlight.rb | 13 +------------ lib/gitlab/highlight.rb | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 lib/gitlab/highlight.rb (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index e76a6f27856..f34eff62d79 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -20,18 +20,7 @@ module Gitlab blob = repository.blob_at(ref, file_name) return [] unless blob - content = blob.data - lexer = Rouge::Lexer.guess(filename: file_name, source: content).new rescue Rouge::Lexers::PlainText.new - formatter.format(lexer.lex(content)).lines.map!(&:html_safe) - end - - def self.formatter - @formatter ||= Rouge::Formatters::HTMLGitlab.new( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + Gitlab::Highlight.highlight(file_name, blob.data).lines.map!(&:html_safe) end def initialize(diff_file) diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb new file mode 100644 index 00000000000..02e097eca3d --- /dev/null +++ b/lib/gitlab/highlight.rb @@ -0,0 +1,23 @@ +module Gitlab + class Highlight + def self.highlight(blob_name, blob_content, nowrap: true, continue: false) + formatter = rouge_formatter(nowrap: nowrap) + + lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + formatter.format(lexer.lex(blob_content, continue: continue)).html_safe + end + + private + + def self.rouge_formatter(options = {}) + options = options.reverse_merge( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + + Rouge::Formatters::HTMLGitlab.new(options) + end + end +end -- cgit v1.2.1 From 83e4fc188b22731d89106b4da28f11bf5509c116 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 16:13:35 +0100 Subject: Refactor highlighting lines --- lib/gitlab/diff/highlight.rb | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f34eff62d79..ba2f12db147 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -32,23 +32,8 @@ module Gitlab def highlight return [] if @lines.empty? - extract_line_prefixes - update_diff_lines - end - - private - - def text_lines - @text_lines ||= @lines.map(&:text) - end - - def extract_line_prefixes - @diff_line_prefixes ||= text_lines.map { |line| line.sub!(/\A((\+|\-))/, '');$1 } - end - - def update_diff_lines @lines.each_with_index do |line, i| - line_prefix = @diff_line_prefixes[i] || ' ' + line_prefix = line.text.match(/\A([+-])/) ? $1 : ' ' # ignore highlighting for "match" lines next if line.type == 'match' @@ -62,24 +47,18 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.sub(/\A\s/, line_prefix).html_safe if highlighted_line + line.text = highlighted_line.insert(0, line_prefix).html_safe if highlighted_line end @lines end def old_lines - @old_lines ||= begin - lines = self.class.process_file(diff_repository, diff_old_ref, diff_old_path) - lines.map! { |line| " #{line}" } - end + @old_lines ||= self.class.process_file(diff_repository, diff_old_ref, diff_old_path) end def new_lines - @new_lines ||= begin - lines = self.class.process_file(diff_repository, diff_new_ref, diff_new_path) - lines.map! { |line| " #{line}" } - end + @new_lines ||= self.class.process_file(diff_repository, diff_new_ref, diff_new_path) end end end -- cgit v1.2.1 From ac652d82f17d378e485dcef15a8fabdcf9bad76b Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 14 Jan 2016 19:45:43 +0100 Subject: Let the CI runner know about builds that this build depends on This allows us to implement artifacts passing: runner will download artifacts from all prior builds --- lib/ci/api/builds.rb | 4 ++-- lib/ci/api/entities.rb | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index fb87637b94f..690bbf97a89 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -20,7 +20,7 @@ module Ci if build update_runner_info - present build, with: Entities::Build + present build, with: Entities::BuildDetails else not_found! end @@ -111,7 +111,7 @@ module Ci build.artifacts_metadata = metadata if build.save - present(build, with: Entities::Build) + present(build, with: Entities::BuildDetails) else render_validation_error!(build) end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index e4ac0545ea2..835f8c97021 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -16,10 +16,19 @@ module Ci end class Build < Grape::Entity - expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, - :before_sha, :allow_git_fetch, :project_name - + expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage + expose :project_id + expose :project_name + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts_file.exists? } + end + + class BuildDetails < Build + expose :commands + expose :repo_url + expose :before_sha + expose :allow_git_fetch + expose :token expose :options do |model| model.options @@ -30,7 +39,9 @@ module Ci end expose :variables - expose :artifacts_file, using: ArtifactFile + expose :dependencies do + expose :depends_on_builds, as: :builds, using: Build + end end class Runner < Grape::Entity -- cgit v1.2.1 From 8dfad143d44af4896ff6c71e8a42ad32b69ad593 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 14 Jan 2016 22:28:07 +0100 Subject: Add inline diff markers in highlighted diffs. --- lib/gitlab/diff/file.rb | 2 +- lib/gitlab/diff/highlight.rb | 206 ++++++++++++++++++++++++++++++++++++------- lib/gitlab/highlight.rb | 7 ++ 3 files changed, 181 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index cb93c6a574a..c1a6e16da5a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -18,7 +18,7 @@ module Gitlab end def highlighted_diff_lines - Gitlab::Diff::Highlight.process_diff_lines(self) + Gitlab::Diff::Highlight.new(self).highlight end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index ba2f12db147..e21f496102d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -6,59 +6,199 @@ module Gitlab delegate :repository, :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - # Apply syntax highlight to provided source code - # - # diff_file - an instance of Gitlab::Diff::File - # - # Returns an Array with the processed items. - def self.process_diff_lines(diff_file) - processor = new(diff_file) - processor.highlight + def initialize(diff_file) + @diff_file = diff_file + @diff_lines = diff_file.diff_lines + @raw_lines = @diff_lines.map(&:text) end - def self.process_file(repository, ref, file_name) - blob = repository.blob_at(ref, file_name) - return [] unless blob + def highlight + return [] if @diff_lines.empty? - Gitlab::Highlight.highlight(file_name, blob.data).lines.map!(&:html_safe) - end + find_inline_diffs - def initialize(diff_file) - @diff_file = diff_file - @file_name = diff_file.new_path - @lines = diff_file.diff_lines + process_lines + + @diff_lines end - def highlight - return [] if @lines.empty? + private - @lines.each_with_index do |line, i| - line_prefix = line.text.match(/\A([+-])/) ? $1 : ' ' + def find_inline_diffs + @inline_diffs = [] + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = @raw_lines[old_index][1..-1] + new_line = @raw_lines[new_index][1..-1] + + # Skip inline diff if empty line was replaced with content + next if old_line == "" + + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line, new_line) + + old_diff_range = lcp..(old_line.length - lcs - 1) + new_diff_range = lcp..(new_line.length - lcs - 1) + + @inline_diffs[old_index] = old_diff_range if old_diff_range.begin <= old_diff_range.end + @inline_diffs[new_index] = new_diff_range if new_diff_range.begin <= new_diff_range.end + end + end + + def process_lines + @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines - next if line.type == 'match' + next if diff_line.type == 'match' + + rich_line = highlight_line(diff_line, i) + rich_line = mark_inline_diffs(rich_line, diff_line, i) + diff_line.text = rich_line.html_safe + end + end + + def highlight_line(diff_line, index) + line_prefix = line_prefixes[index] + + case diff_line.type + when 'new', nil + rich_line = new_lines[diff_line.new_pos - 1] + when 'old' + rich_line = old_lines[diff_line.old_pos - 1] + end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + rich_line ? line_prefix + rich_line : diff_line.text + end + + def mark_inline_diffs(rich_line, diff_line, index) + inline_diff = @inline_diffs[index] + return rich_line unless inline_diff + + raw_line = diff_line.text + + # Based on the prefixless versions + from = inline_diff.begin + 1 + to = inline_diff.end + 1 + + position_mapping = map_character_positions(raw_line, rich_line) + inline_diff_positions = position_mapping[from..to] + marker_ranges = collapse_ranges(inline_diff_positions) + + offset = 0 + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) + end + + rich_line + end - case line.type - when 'new', nil - highlighted_line = new_lines[line.new_pos - 1] - when 'old' - highlighted_line = old_lines[line.old_pos - 1] + def line_prefixes + @line_prefixes ||= @raw_lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + end + + def local_edit_indexes + @local_edit_indexes ||= begin + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 end - # Only update text if line is found. This will prevent - # issues with submodules given the line only exists in diff content. - line.text = highlighted_line.insert(0, line_prefix).html_safe if highlighted_line + local_edit_indexes end + end + + def map_character_positions(raw_line, rich_line) + mapping = [] + raw_pos = 0 + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + raw_char = raw_line[raw_pos] + rich_char = rich_line[rich_pos] + + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end - @lines + mapping[raw_pos] = rich_pos + + rich_pos += 1 + end + + mapping end def old_lines - @old_lines ||= self.class.process_file(diff_repository, diff_old_ref, diff_old_path) + @old_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_old_ref, diff_old_path) end def new_lines - @new_lines ||= self.class.process_file(diff_repository, diff_new_ref, diff_new_path) + @new_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_new_ref, diff_new_path) + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + def insert_around_range(text, range, before, after, offset = 0) + from = range.begin + to = range.end + + text.insert(offset + from, before) + offset += before.length + + text.insert(offset + to + 1, after) + offset += after.length + + offset end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 02e097eca3d..a5b041687e3 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -7,6 +7,13 @@ module Gitlab formatter.format(lexer.lex(blob_content, continue: continue)).html_safe end + def self.highlight_lines(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + highlight(file_name, blob.data).lines.map!(&:html_safe) + end + private def self.rouge_formatter(options = {}) -- cgit v1.2.1 From 6b9c730e91962a6d6343bcb7fc4dc75c99b41bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 14 Jan 2016 16:38:37 -0500 Subject: More refactoring from last code review. #3945 * Use commit objects instead of IDs when generating diffs * Use proper references when generating MR's source and target * Update broken specs --- lib/gitlab/diff/file.rb | 5 ++--- lib/gitlab/diff/highlight.rb | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c1a6e16da5a..a6a7fc8ff4c 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,14 +1,13 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :new_ref, :old_ref + attr_reader :diff, :new_ref, :old_ref delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff, diff_refs, repository) + def initialize(diff, diff_refs) @diff = diff - @repository = repository @old_ref, @new_ref = diff_refs end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index e21f496102d..fb79a2a69de 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,8 +3,7 @@ module Gitlab class Highlight attr_reader :diff_file - delegate :repository, :old_path, :new_path, :old_ref, :new_ref, - to: :diff_file, prefix: :diff + delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff def initialize(diff_file) @diff_file = diff_file @@ -141,11 +140,11 @@ module Gitlab end def old_lines - @old_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_old_ref, diff_old_path) + @old_lines ||= self.class.process_file(*processing_args(:old)) end def new_lines - @new_lines ||= Gitlab::Highlight.highlight_lines(diff_repository, diff_new_ref, diff_new_path) + @new_lines ||= self.class.process_file(*processing_args(:new)) end def longest_common_suffix(a, b) @@ -200,6 +199,16 @@ module Gitlab offset end + + private + + def processing_args(version) + ref = send("diff_#{version}_ref") + path = send("diff_#{version}_path") + + [ref.project.repository, ref.id, path] + end + end end end -- cgit v1.2.1 From 71b4341a37dc274ce94d0b967e5b69fd49834cb8 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:11:36 +0100 Subject: Method was moved --- lib/gitlab/diff/highlight.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index fb79a2a69de..65deea9d335 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -140,11 +140,11 @@ module Gitlab end def old_lines - @old_lines ||= self.class.process_file(*processing_args(:old)) + @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) end def new_lines - @new_lines ||= self.class.process_file(*processing_args(:new)) + @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end def longest_common_suffix(a, b) -- cgit v1.2.1 From 7d31f372191c38e5676cd6ec746721eb64b4a634 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:25:29 +0100 Subject: Move inline diff logic to its own class --- lib/gitlab/diff/highlight.rb | 84 ++++++-------------------------------------- 1 file changed, 11 insertions(+), 73 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 65deea9d335..f89121e184b 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,26 +24,7 @@ module Gitlab private def find_inline_diffs - @inline_diffs = [] - - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 - old_line = @raw_lines[old_index][1..-1] - new_line = @raw_lines[new_index][1..-1] - - # Skip inline diff if empty line was replaced with content - next if old_line == "" - - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line, new_line) - - old_diff_range = lcp..(old_line.length - lcs - 1) - new_diff_range = lcp..(new_line.length - lcs - 1) - - @inline_diffs[old_index] = old_diff_range if old_diff_range.begin <= old_diff_range.end - @inline_diffs[new_index] = new_diff_range if new_diff_range.begin <= new_diff_range.end - end + @inline_diffs = InlineDiff.new(@raw_lines).inline_diffs end def process_lines @@ -58,7 +39,7 @@ module Gitlab end def highlight_line(diff_line, index) - line_prefix = line_prefixes[index] + line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' case diff_line.type when 'new', nil @@ -73,44 +54,23 @@ module Gitlab end def mark_inline_diffs(rich_line, diff_line, index) - inline_diff = @inline_diffs[index] - return rich_line unless inline_diff + line_inline_diffs = @inline_diffs[index] + return rich_line unless line_inline_diffs raw_line = diff_line.text - - # Based on the prefixless versions - from = inline_diff.begin + 1 - to = inline_diff.end + 1 - position_mapping = map_character_positions(raw_line, rich_line) - inline_diff_positions = position_mapping[from..to] - marker_ranges = collapse_ranges(inline_diff_positions) offset = 0 - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end - - rich_line - end - - def line_prefixes - @line_prefixes ||= @raw_lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - end - - def local_edit_indexes - @local_edit_indexes ||= begin - joined_line_prefixes = " #{line_prefixes.join} " + line_inline_diffs.each do |inline_diff_range| + inline_diff_positions = position_mapping[inline_diff_range] + marker_ranges = collapse_ranges(inline_diff_positions) - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) end - - local_edit_indexes end + + rich_line end def map_character_positions(raw_line, rich_line) @@ -147,25 +107,6 @@ module Gitlab @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end - def longest_common_suffix(a, b) - longest_common_prefix(a.reverse, b.reverse) - end - - def longest_common_prefix(a, b) - max_length = [a.length, b.length].max - - length = 0 - (0..max_length - 1).each do |pos| - old_char = a[pos] - new_char = b[pos] - - break if old_char != new_char - length += 1 - end - - length - end - def collapse_ranges(positions) return [] if positions.empty? ranges = [] @@ -200,15 +141,12 @@ module Gitlab offset end - private - def processing_args(version) ref = send("diff_#{version}_ref") path = send("diff_#{version}_path") [ref.project.repository, ref.id, path] end - end end end -- cgit v1.2.1 From 13f10efcf120f2d5244bf6abe934dc7c026834ef Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 14:39:43 +0100 Subject: Move inline diff marker logic to its own class --- lib/gitlab/diff/highlight.rb | 106 +++++-------------------------------------- 1 file changed, 11 insertions(+), 95 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index f89121e184b..b6875f07279 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -12,32 +12,24 @@ module Gitlab end def highlight - return [] if @diff_lines.empty? - - find_inline_diffs - - process_lines - - @diff_lines - end - - private - - def find_inline_diffs - @inline_diffs = InlineDiff.new(@raw_lines).inline_diffs - end - - def process_lines @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines next if diff_line.type == 'match' rich_line = highlight_line(diff_line, i) - rich_line = mark_inline_diffs(rich_line, diff_line, i) + + if line_inline_diffs = inline_diffs[i] + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + end + diff_line.text = rich_line.html_safe end + + @diff_lines end + private + def highlight_line(diff_line, index) line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' @@ -53,50 +45,8 @@ module Gitlab rich_line ? line_prefix + rich_line : diff_line.text end - def mark_inline_diffs(rich_line, diff_line, index) - line_inline_diffs = @inline_diffs[index] - return rich_line unless line_inline_diffs - - raw_line = diff_line.text - position_mapping = map_character_positions(raw_line, rich_line) - - offset = 0 - line_inline_diffs.each do |inline_diff_range| - inline_diff_positions = position_mapping[inline_diff_range] - marker_ranges = collapse_ranges(inline_diff_positions) - - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end - end - - rich_line - end - - def map_character_positions(raw_line, rich_line) - mapping = [] - raw_pos = 0 - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - raw_char = raw_line[raw_pos] - rich_char = rich_line[rich_pos] - - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = rich_pos - - rich_pos += 1 - end - - mapping + def inline_diffs + @inline_diffs ||= InlineDiff.new(@raw_lines).inline_diffs end def old_lines @@ -107,40 +57,6 @@ module Gitlab @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - def insert_around_range(text, range, before, after, offset = 0) - from = range.begin - to = range.end - - text.insert(offset + from, before) - offset += before.length - - text.insert(offset + to + 1, after) - offset += after.length - - offset - end - def processing_args(version) ref = send("diff_#{version}_ref") path = send("diff_#{version}_path") -- cgit v1.2.1 From 1ce766367eb529fe88068be2f34315f87d74a349 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 15 Jan 2016 15:35:33 +0100 Subject: Change dependencies.builds to depends_on_builds --- lib/ci/api/entities.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 835f8c97021..b25e0e573a8 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts_file.exists? } + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } end class BuildDetails < Build @@ -39,9 +39,7 @@ module Ci end expose :variables - expose :dependencies do - expose :depends_on_builds, as: :builds, using: Build - end + expose :depends_on_builds, using: Build end class Runner < Grape::Entity -- cgit v1.2.1 From 5de8971ddfa10cbc6a082316eee7377038670f75 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 16:02:48 +0100 Subject: Whoops, forgot to add files --- lib/gitlab/diff/inline_diff.rb | 75 +++++++++++++++++++++++++++++++ lib/gitlab/diff/inline_diff_marker.rb | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 lib/gitlab/diff/inline_diff.rb create mode 100644 lib/gitlab/diff/inline_diff_marker.rb (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb new file mode 100644 index 00000000000..bae297ab00f --- /dev/null +++ b/lib/gitlab/diff/inline_diff.rb @@ -0,0 +1,75 @@ +module Gitlab + module Diff + class InlineDiff + attr_accessor :lines + + def initialize(lines) + @lines = lines + end + + def inline_diffs + inline_diffs = [] + + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = @lines[old_index] + new_line = @lines[new_index] + + suffixless_old_line = old_line[1..-1] + suffixless_new_line = new_line[1..-1] + + # Skip inline diff if empty line was replaced with content + next if suffixless_old_line == "" + + # Add one, because this is based on the suffixless version + lcp = longest_common_prefix(suffixless_old_line, suffixless_new_line) + 1 + lcs = longest_common_suffix(suffixless_old_line, suffixless_new_line) + + old_diff_range = lcp..(old_line.length - lcs - 1) + new_diff_range = lcp..(new_line.length - lcs - 1) + + inline_diffs[old_index] = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + inline_diffs[new_index] = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + end + + inline_diffs + end + + private + + def local_edit_indexes + line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 + end + + local_edit_indexes + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb new file mode 100644 index 00000000000..4bb755e3d3d --- /dev/null +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -0,0 +1,85 @@ +module Gitlab + module Diff + class InlineDiffMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = rich_line + end + + def mark(line_inline_diffs) + offset = 0 + line_inline_diffs.each do |inline_diff_range| + inline_diff_positions = position_mapping[inline_diff_range] + marker_ranges = collapse_ranges(inline_diff_positions) + + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) + end + end + + rich_line + end + + def position_mapping + @position_mapping ||= begin + mapping = [] + raw_pos = 0 + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + raw_char = raw_line[raw_pos] + rich_char = rich_line[rich_pos] + + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = rich_pos + + rich_pos += 1 + end + + mapping + end + end + + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + def insert_around_range(text, range, before, after, offset = 0) + text.insert(offset + range.begin, before) + offset += before.length + + text.insert(offset + range.end + 1, after) + offset += after.length + + offset + end + end + end +end -- cgit v1.2.1 From 12546f895f86929ffe69299a02c93692370ee55a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 15 Jan 2016 16:29:26 +0100 Subject: Add tests --- lib/gitlab/diff/inline_diff.rb | 1 + lib/gitlab/diff/inline_diff_marker.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index bae297ab00f..e5986fd69e2 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -38,6 +38,7 @@ module Gitlab private + # Find runs of single line edits def local_edit_indexes line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } joined_line_prefixes = " #{line_prefixes.join} " diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 4bb755e3d3d..405465a641f 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -11,9 +11,12 @@ module Gitlab def mark(line_inline_diffs) offset = 0 line_inline_diffs.each do |inline_diff_range| + # Map the inline-diff range based on the raw line to character positions in the rich line inline_diff_positions = position_mapping[inline_diff_range] + # Turn the array of character positions into ranges marker_ranges = collapse_ranges(inline_diff_positions) + # Mark each range marker_ranges.each do |range| offset = insert_around_range(rich_line, range, "", "", offset) end @@ -22,6 +25,9 @@ module Gitlab rich_line end + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin mapping = [] @@ -31,6 +37,8 @@ module Gitlab raw_char = raw_line[raw_pos] rich_char = rich_line[rich_pos] + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment while rich_char == '<' until rich_char == '>' rich_pos += 1 @@ -50,6 +58,7 @@ module Gitlab end end + # Takes an array of integers, and returns an array of ranges covering the same integers def collapse_ranges(positions) return [] if positions.empty? ranges = [] @@ -71,6 +80,7 @@ module Gitlab ranges end + # Inserts tags around the characters identified by the given range def insert_around_range(text, range, before, after, offset = 0) text.insert(offset + range.begin, before) offset += before.length -- cgit v1.2.1 From e8785ded1e922a72989614d7e70e9c26c367f1aa Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 15 Jan 2016 17:47:28 -0500 Subject: Prevent StateMachine warnings from outputting during a cron task [ci skip] Closes #5931 --- lib/tasks/gitlab/task_helpers.rake | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index 8c63877e51c..d33b5b31e18 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -4,6 +4,9 @@ end String.disable_colorization = true unless STDOUT.isatty +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + namespace :gitlab do # Ask if the user wants to continue -- cgit v1.2.1 From bddef293d666c4b3933b93b017ae49bdf18b234f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 18 Jan 2016 22:43:04 -0200 Subject: Ensure that doesn't have pending migrations when running the specs --- lib/gitlab/current_settings.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7f938780ab1..ea054255820 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -39,7 +39,6 @@ module Gitlab end use_db && ActiveRecord::Base.connection.active? && - !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError -- cgit v1.2.1 From 0a8039eb7790426880bdd7b9d67775aeb6e5dac7 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 13:56:36 +0100 Subject: Remove useless assignments --- lib/gitlab/diff/inline_diff_marker.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 405465a641f..8998ccba5ce 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -31,10 +31,8 @@ module Gitlab def position_mapping @position_mapping ||= begin mapping = [] - raw_pos = 0 rich_pos = 0 (0..raw_line.length).each do |raw_pos| - raw_char = raw_line[raw_pos] rich_char = rich_line[rich_pos] # The raw and rich lines are the same except for HTML tags, -- cgit v1.2.1 From 512bebe21d7f57b691a1c8355581feb64b9b6292 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 14:52:41 +0100 Subject: Refactor Gitlab::Highlight and fix tests --- lib/gitlab/highlight.rb | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index a5b041687e3..28cfebef968 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,10 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, continue: false) - formatter = rouge_formatter(nowrap: nowrap) - - lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText - formatter.format(lexer.lex(blob_content, continue: continue)).html_safe + def self.highlight(blob_name, blob_content, nowrap: true) + new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) end def self.highlight_lines(repository, ref, file_name) @@ -14,9 +11,26 @@ module Gitlab highlight(file_name, blob.data).lines.map!(&:html_safe) end + def initialize(blob_name, blob_content, nowrap: true) + @formatter = rouge_formatter(nowrap: nowrap) + @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + end + + def highlight(text, continue: true) + @formatter.format(lex(text, continue: continue)).html_safe + end + private - def self.rouge_formatter(options = {}) + def lex(text, continue: true) + if @lexer == Rouge::Lexers::PlainText + @lexer.lex(text) + else + @lexer.lex(text, continue: continue) + end + end + + def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, cssclass: 'code highlight', -- cgit v1.2.1 From 5c7259c7c30228bf85a4efbbddb673046e8d733f Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 15:13:37 +0100 Subject: Don't crash when file can't be highlighted --- lib/gitlab/highlight.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 28cfebef968..4ddb4fea977 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -17,19 +17,13 @@ module Gitlab end def highlight(text, continue: true) - @formatter.format(lex(text, continue: continue)).html_safe + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + rescue + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end private - def lex(text, continue: true) - if @lexer == Rouge::Lexers::PlainText - @lexer.lex(text) - else - @lexer.lex(text, continue: continue) - end - end - def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, -- cgit v1.2.1 From 10669097fa7621a175755d144c61ecb80a5ea9fb Mon Sep 17 00:00:00 2001 From: Josh Frye Date: Wed, 13 Jan 2016 09:07:21 -0500 Subject: Add public params to GET /projects api. Closes #3788 --- lib/api/helpers.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6d2380cf47d..68d0aa671b6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,6 +264,10 @@ module API projects = projects.search(params[:search]) end + if params[:public].present? && parse_boolean(params[:public]) + projects = projects.public_only + end + projects.reorder(project_order_by => project_sort) end -- cgit v1.2.1 From 11797df1af483156b8cf11290c49c3f4d6089d99 Mon Sep 17 00:00:00 2001 From: Josh Frye Date: Mon, 18 Jan 2016 12:41:49 -0500 Subject: Change 'public' flag to more robust 'visibility' query. --- lib/api/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 68d0aa671b6..3f528b9f7c0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,8 +264,8 @@ module API projects = projects.search(params[:search]) end - if params[:public].present? && parse_boolean(params[:public]) - projects = projects.public_only + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) end projects.reorder(project_order_by => project_sort) -- cgit v1.2.1 From 98e1a5b63424b6912de98ee5055d3f9e57e63899 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 19 Jan 2016 16:25:38 +0100 Subject: Allow LDAP users to change their email if it was not set by the LDAP server --- lib/gitlab/ldap/user.rb | 29 ++++++++++++++++------------- lib/gitlab/o_auth/auth_hash.rb | 8 ++++++-- lib/gitlab/o_auth/user.rb | 14 +++++++------- 3 files changed, 29 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index aef08c97d1d..e044f0ecc6d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -30,28 +30,31 @@ module Gitlab end def find_by_uid_and_provider - self.class.find_by_uid_and_provider( - auth_hash.uid, auth_hash.provider) + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? end def update_user_attributes - return unless persisted? + if persisted? + if auth_hash.has_email? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) + # For a new identity set extern_uid to the LDAP DN + # For an existing identity with matching email but changed DN, update the DN. + # For an existing identity with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid + end - # For a new user set extern_uid to the LDAP DN - # For an existing user with matching email but changed DN, update the DN. - # For an existing user with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid + gl_user.ldap_email = auth_hash.has_email? gl_user end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index ba31599432b..36e5c2670bb 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,6 +32,10 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end + def has_email? + get_info(:email).present? + end + private def info @@ -46,8 +50,8 @@ module Gitlab def username_and_email @username_and_email ||= begin - username = get_info(:username) || get_info(:nickname) - email = get_info(:email) + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence username ||= generate_username(email) if email email ||= generate_temporarily_email(username) if username diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e3d2cc65a8f..d87a72f7ba3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -111,7 +111,7 @@ module Gitlab def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users - else + else Gitlab.config.omniauth.block_auto_created_users end end @@ -135,16 +135,16 @@ module Gitlab def user_attributes # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? - username = ldap_person.username - email = ldap_person.email.first - else - username = auth_hash.username - email = auth_hash.email + username = ldap_person.username.presence + email = ldap_person.email.first.presence end + username ||= auth_hash.username + email ||= auth_hash.email + name = auth_hash.name name = ::Namespace.clean_path(username) if name.strip.empty? - + { name: name, username: ::Namespace.clean_path(username), -- cgit v1.2.1 From 84124380e815e549f50d82051b2b546c12d9a724 Mon Sep 17 00:00:00 2001 From: Anton Baklanov Date: Sat, 19 Dec 2015 21:04:40 +0200 Subject: Added X-GitLab-... headers to emails from CI and Email On Push services Fixes #2098 --- lib/gitlab/email/message/repository_push.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index a2eb7a70bd2..a05ffeb9cd2 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -9,6 +9,7 @@ module Gitlab delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author + delegate :username, to: :author, prefix: :author def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless -- cgit v1.2.1 From 701513dcc7afb403372bc738642a9a52e9be5983 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 14:51:56 +0100 Subject: Move parallel diff logic to separate class --- lib/gitlab/diff/file.rb | 4 ++ lib/gitlab/diff/parallel_diff.rb | 117 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 lib/gitlab/diff/parallel_diff.rb (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index a6a7fc8ff4c..74b1c117129 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -20,6 +20,10 @@ module Gitlab Gitlab::Diff::Highlight.new(self).highlight end + def parallel_diff_lines + Gitlab::Diff::ParallelDiff.new(self).parallelize + end + def mode_changed? !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb new file mode 100644 index 00000000000..a0dc3da875d --- /dev/null +++ b/lib/gitlab/diff/parallel_diff.rb @@ -0,0 +1,117 @@ +module Gitlab + module Diff + class ParallelDiff + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + def parallelize + lines = [] + skip_next = false + + diff_file.highlighted_diff_lines.each do |line| + full_line = line.text + type = line.type + line_code = generate_line_code(diff_file.file_path, line) + line_new = line.new_pos + line_old = line.old_pos + + next_line = diff_file.next_line(line.index) + + if next_line + next_line_code = generate_line_code(diff_file.file_path, next_line) + next_type = next_line.type + next_line = next_line.text + end + + case type + when 'match', nil + # line in the right panel is the same as in the left one + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + when 'old' + case next_type + when 'new' + # Left side has text removed, right side has text added + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: line_new, + text: next_line, + line_code: next_line_code + } + } + skip_next = true + when 'old', nil + # Left side has text removed, right side doesn't have any change + # No next line code, no new line number, no new line text + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: nil, + text: "", + line_code: nil + } + } + end + when 'new' + if skip_next + # Change has been already included in previous line so no need to do it again + skip_next = false + next + else + # Change is only on the right side, left side has no change + lines << { + left: { + type: nil, + number: nil, + text: "", + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + end + end + end + lines + end + + private + + def generate_line_code(file_path, line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + end + end +end -- cgit v1.2.1 From a010db5db243a532cb8d1c2d5ac787e90da0044f Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 15:26:44 +0100 Subject: Properly handle HTML entities with inline diffs --- lib/gitlab/diff/inline_diff.rb | 11 ++++------- lib/gitlab/diff/inline_diff_marker.rb | 16 ++++++++++++++-- lib/gitlab/diff/parser.rb | 7 +------ 3 files changed, 19 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index e5986fd69e2..b8a61ad6115 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -16,15 +16,12 @@ module Gitlab old_line = @lines[old_index] new_line = @lines[new_index] - suffixless_old_line = old_line[1..-1] - suffixless_new_line = new_line[1..-1] - # Skip inline diff if empty line was replaced with content - next if suffixless_old_line == "" + next if old_line[1..-1] == "" - # Add one, because this is based on the suffixless version - lcp = longest_common_prefix(suffixless_old_line, suffixless_new_line) + 1 - lcs = longest_common_suffix(suffixless_old_line, suffixless_new_line) + # Add one, because this is based on the prefixless version + lcp = longest_common_prefix(old_line[1..-1], new_line[1..-1]) + 1 + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) old_diff_range = lcp..(old_line.length - lcs - 1) new_diff_range = lcp..(new_line.length - lcs - 1) diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 8998ccba5ce..c31149d374e 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -12,7 +12,7 @@ module Gitlab offset = 0 line_inline_diffs.each do |inline_diff_range| # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range] + inline_diff_positions = position_mapping[inline_diff_range].flatten # Turn the array of character positions into ranges marker_ranges = collapse_ranges(inline_diff_positions) @@ -47,7 +47,19 @@ module Gitlab rich_char = rich_line[rich_pos] end - mapping[raw_pos] = rich_pos + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end rich_pos += 1 end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 2c0a866e8ba..6c8a1fc6d6f 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -14,7 +14,7 @@ module Gitlab @lines.each do |line| next if filename?(line) - full_line = html_escape(line.gsub(/\n/, '')) + full_line = line.gsub(/\n/, '') if line.match(/^@@ -/) type = "match" @@ -67,11 +67,6 @@ module Gitlab nil end end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end -- cgit v1.2.1 From 8536e083f7b2d7ed77ecae83774d75f68d66e0b4 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 20 Jan 2016 13:16:34 +0100 Subject: Add IP blocking against DNSBL at sign-up --- lib/dnsxl_check.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/ip_check.rb | 34 ++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 lib/dnsxl_check.rb create mode 100644 lib/gitlab/ip_check.rb (limited to 'lib') diff --git a/lib/dnsxl_check.rb b/lib/dnsxl_check.rb new file mode 100644 index 00000000000..1e506b2d9cb --- /dev/null +++ b/lib/dnsxl_check.rb @@ -0,0 +1,105 @@ +require 'resolv' + +class DNSXLCheck + + class Resolver + def self.search(query) + begin + Resolv.getaddress(query) + true + rescue Resolv::ResolvError + false + end + end + end + + IP_REGEXP = /\A(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\z/ + DEFAULT_THRESHOLD = 0.33 + + def self.create_from_list(list) + dnsxl_check = DNSXLCheck.new + + list.each do |entry| + dnsxl_check.add_list(entry.domain, entry.weight) + end + + dnsxl_check + end + + def test(ip) + if use_threshold? + test_with_threshold(ip) + else + test_strict(ip) + end + end + + def test_with_threshold(ip) + return false if lists.empty? + + search(ip) + final_score >= threshold + end + + def test_strict(ip) + return false if lists.empty? + + search(ip) + @score > 0 + end + + def use_threshold=(value) + @use_threshold = value == true + end + + def use_threshold? + @use_threshold &&= true + end + + def threshold=(threshold) + raise ArgumentError, "'threshold' value must be grather than 0 and less than or equal to 1" unless threshold > 0 && threshold <= 1 + @threshold = threshold + end + + def threshold + @threshold ||= DEFAULT_THRESHOLD + end + + def add_list(domain, weight) + @lists ||= [] + @lists << { domain: domain, weight: weight } + end + + def lists + @lists ||= [] + end + + private + + def search(ip) + raise ArgumentError, "'ip' value must be in #{IP_REGEXP} format" unless ip.match(IP_REGEXP) + + @score = 0 + + reversed = reverse_ip(ip) + search_in_rbls(reversed) + end + + def reverse_ip(ip) + ip.split('.').reverse.join('.') + end + + def search_in_rbls(reversed_ip) + lists.each do |rbl| + query = "#{reversed_ip}.#{rbl[:domain]}" + @score += rbl[:weight] if Resolver.search(query) + end + end + + def final_score + weights = lists.map{ |rbl| rbl[:weight] }.reduce(:+).to_i + return 0 if weights == 0 + + (@score.to_f / weights.to_f).round(2) + end +end diff --git a/lib/gitlab/ip_check.rb b/lib/gitlab/ip_check.rb new file mode 100644 index 00000000000..f2e9b50d225 --- /dev/null +++ b/lib/gitlab/ip_check.rb @@ -0,0 +1,34 @@ +module Gitlab + class IpCheck + + def initialize(ip) + @ip = ip + + application_settings = ApplicationSetting.current + @ip_blocking_enabled = application_settings.ip_blocking_enabled + @dnsbl_servers_list = application_settings.dnsbl_servers_list + end + + def spam? + @ip_blocking_enabled && blacklisted? + end + + private + + def blacklisted? + on_dns_blacklist? + end + + def on_dns_blacklist? + dnsbl_check = DNSXLCheck.new + prepare_dnsbl_list(dnsbl_check) + dnsbl_check.test(@ip) + end + + def prepare_dnsbl_list(dnsbl_check) + @dnsbl_servers_list.split(',').map(&:strip).reject(&:empty?).each do |domain| + dnsbl_check.add_list(domain, 1) + end + end + end +end -- cgit v1.2.1 From a10ab94b068c31601c7d4ab0062b9d567af6cee2 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 13 Jan 2016 07:05:28 -0800 Subject: Gracefully handle invalid UTF-8 sequences in Markdown links Closes #6077 --- lib/banzai/filter/reference_filter.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 20bd4f7ee6e..3637b1bac94 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,6 +133,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,6 +171,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? next unless link && link =~ /\A#{pattern}\z/ html = yield link, text -- cgit v1.2.1 From 577f2fb47a7ef57415b78b49d5c746d4e99f6a98 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 18:44:27 +0100 Subject: Save and use actual diff base commit for MR diff highlighting --- lib/gitlab/diff/file.rb | 12 ++++++++++-- lib/gitlab/diff/highlight.rb | 9 ++++++++- lib/gitlab/diff/inline_diff_marker.rb | 13 +++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 74b1c117129..a484177ae33 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,14 +1,22 @@ module Gitlab module Diff class File - attr_reader :diff, :new_ref, :old_ref + attr_reader :diff, :diff_refs delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false def initialize(diff, diff_refs) @diff = diff - @old_ref, @new_ref = diff_refs + @diff_refs = diff_refs + end + + def old_ref + diff_refs[0] if diff_refs + end + + def new_ref + diff_refs[1] if diff_refs end # Array of Gitlab::DIff::Line objects diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b6875f07279..fe084a7399a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -31,6 +31,8 @@ module Gitlab private def highlight_line(diff_line, index) + return html_escape(diff_line.text) unless diff_file.diff_refs + line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' case diff_line.type @@ -42,7 +44,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - rich_line ? line_prefix + rich_line : diff_line.text + rich_line ? line_prefix + rich_line : html_escape(diff_line.text) end def inline_diffs @@ -63,6 +65,11 @@ module Gitlab [ref.project.repository, ref.id, path] end + + def html_escape(str) + replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } + str.gsub(/[&"'><]/, replacements) + end end end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index c31149d374e..7ca198eeb86 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -9,17 +9,18 @@ module Gitlab end def mark(line_inline_diffs) - offset = 0 + marker_ranges = [] line_inline_diffs.each do |inline_diff_range| # Map the inline-diff range based on the raw line to character positions in the rich line inline_diff_positions = position_mapping[inline_diff_range].flatten # Turn the array of character positions into ranges - marker_ranges = collapse_ranges(inline_diff_positions) + marker_ranges.concat(collapse_ranges(inline_diff_positions)) + end - # Mark each range - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) - end + offset = 0 + # Mark each range + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "", "", offset) end rich_line -- cgit v1.2.1 From 0e992a3b4e710f8486a37bfa73ad6981365fceb2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 19:20:13 +0100 Subject: Properly highlight lines around '\ No newline at end of file' --- lib/gitlab/diff/highlight.rb | 4 ++-- lib/gitlab/diff/parallel_diff.rb | 2 +- lib/gitlab/diff/parser.rb | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index fe084a7399a..179f8164c84 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,7 +14,7 @@ module Gitlab def highlight @diff_lines.each_with_index do |diff_line, i| # ignore highlighting for "match" lines - next if diff_line.type == 'match' + next if diff_line.type == 'match' || diff_line.type == 'nonewline' rich_line = highlight_line(diff_line, i) @@ -33,7 +33,7 @@ module Gitlab def highlight_line(diff_line, index) return html_escape(diff_line.text) unless diff_file.diff_refs - line_prefix = diff_line.text.match(/\A([+-])/) ? $1 : ' ' + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' case diff_line.type when 'new', nil diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index a0dc3da875d..c0db3559e3a 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -62,7 +62,7 @@ module Gitlab } } skip_next = true - when 'old', nil + when 'old', 'nonewline', nil # Left side has text removed, right side doesn't have any change # No next line code, no new line number, no new line text lines << { diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 6c8a1fc6d6f..3666063bf8b 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -26,6 +26,10 @@ module Gitlab lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next + elsif line[0] == '\\' + type = 'nonewline' + lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 else type = identification_type(line) lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) @@ -33,10 +37,13 @@ module Gitlab end - if line[0] == "+" + case line[0] + when "+" line_new += 1 - elsif line[0] == "-" + when "-" line_old += 1 + when "\\" + # No increment else line_new += 1 line_old += 1 @@ -59,9 +66,10 @@ module Gitlab end def identification_type(line) - if line[0] == "+" + case line[0] + when "+" "new" - elsif line[0] == "-" + when "-" "old" else nil -- cgit v1.2.1 From 6000f8545f43b449035cb50382901ce40fb807b0 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 20 Jan 2016 19:33:34 +0100 Subject: Validate bounds just to be sure --- lib/gitlab/diff/inline_diff_marker.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 7ca198eeb86..1d7fa1bce06 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -93,6 +93,9 @@ module Gitlab # Inserts tags around the characters identified by the given range def insert_around_range(text, range, before, after, offset = 0) + # Just to be sure + return offset if offset + range.end + 1 > text.length + text.insert(offset + range.begin, before) offset += before.length -- cgit v1.2.1 From 9b0f57781e8c71eb0e627a63078fedcedfe62bbb Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 20 Jan 2016 21:48:22 +0100 Subject: Add method that calculates total size for artifacts subfolder --- lib/gitlab/ci/build/artifacts/metadata.rb | 8 +++++--- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1344f5d120b..f2020c82d40 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -13,8 +13,8 @@ module Gitlab attr_reader :file, :path, :full_version - def initialize(file, path) - @file, @path = file, path + def initialize(file, path, **opts) + @file, @path, @opts = file, path, opts @full_version = read_version end @@ -52,7 +52,9 @@ module Gitlab def match_entries(gz) entries = {} - match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} + + child_pattern = '[^/]*/?$' unless @opts[:recursive] + match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ until gz.eof? do begin diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 25b71fc3275..7f4c750b6fd 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -95,6 +95,13 @@ module Gitlab children.empty? end + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + def to_s @path end -- cgit v1.2.1 From f4bdbecfd54c39baed79d0143de97933d26e3215 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 20 Jan 2016 19:36:35 -0200 Subject: Fix cross projects detection when importing GitHub pull requests --- lib/gitlab/github_import/pull_request_formatter.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b7c47958cc7..f96fed0f5cf 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -18,7 +18,7 @@ module Gitlab end def cross_project? - source_repo.fork == true + source_repo.id != target_repo.id end def number @@ -73,6 +73,10 @@ module Gitlab project end + def target_repo + raw_data.base.repo + end + def target_branch target_project.repository.find_branch(raw_data.base.ref) end -- cgit v1.2.1 From c45a6bf3ba13cbd532852dfcc48ef3fd7aa545e4 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 14 Jan 2016 17:52:59 +0100 Subject: Added cache:key to .gitlab-ci.yml allowing to fine tune the caching --- lib/ci/gitlab_ci_yaml_processor.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index bcdfd38d292..1a3f662811a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -115,6 +115,10 @@ module Ci end if @cache + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" + end + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) raise ValidationError, "cache:untracked parameter should be an boolean" end @@ -198,6 +202,10 @@ module Ci end def validate_job_cache!(name, job) + if job[:cache][:key] && !validate_string(job[:cache][:key]) + raise ValidationError, "#{name} job: cache:key parameter should be a string" + end + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" end -- cgit v1.2.1 From 693fa7a458645291cc903bf400f9b463941cb63d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 21 Jan 2016 13:36:55 +0100 Subject: Init script changes for gitlab-workhorse in 8.4 --- lib/support/init.d/gitlab | 9 ++++++--- lib/support/init.d/gitlab.default.example | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f07c8b508..1633891c8a0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -38,6 +38,7 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" @@ -233,10 +234,12 @@ start_gitlab() { if [ "$gitlab_workhorse_status" = "0" ]; then echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-workhorse does this itself + # No need to remove a socket, gitlab-workhorse does this itself. + # Because gitlab-workhorse has multiple executables we need to fix + # the PATH. $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ - $app_root/../gitlab-workhorse/gitlab-workhorse \ - $gitlab_workhorse_options \ + /usr/bin/env PATH=$gitlab_workhorse_dir:$PATH \ + gitlab-workhorse $gitlab_workhorse_options \ >> $gitlab_workhorse_log 2>&1 & fi diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1937ca582b0..4e6e56ac2db 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,6 +30,9 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# The directory where the gitlab-workhorse binaries are. Usually +# /home/git/gitlab-workhorse . +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write -- cgit v1.2.1 From 4861b1bbfbd962416452e1e883c67b885678b5a0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Jan 2016 13:11:38 -0200 Subject: Fix import of GitHub's wiki when the repository has not been created --- lib/gitlab/github_import/importer.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 18929b9113b..663402e8197 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -82,8 +82,12 @@ module Gitlab end true - rescue Gitlab::Shell::Error - false + rescue Gitlab::Shell::Error => e + if e.message =~ /repository not exported/ + true + else + false + end end end end -- cgit v1.2.1 From 150b4f66e6a244b5f6bd1119de3ce68336a9bf9e Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 22 Jan 2016 01:43:06 +0100 Subject: Make sure non-highlighted diffs are still escaped --- lib/gitlab/diff/highlight.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 179f8164c84..964c89de6c6 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,13 +1,17 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file + attr_reader :diff_file, :diff_lines, :raw_lines delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - def initialize(diff_file) - @diff_file = diff_file - @diff_lines = diff_file.diff_lines + def initialize(diff_lines) + if diff_lines.is_a?(Gitlab::Diff::File) + @diff_file = diff_file + @diff_lines = diff_file.diff_lines + else + @diff_lines = diff_lines + end @raw_lines = @diff_lines.map(&:text) end @@ -31,7 +35,7 @@ module Gitlab private def highlight_line(diff_line, index) - return html_escape(diff_line.text) unless diff_file.diff_refs + return html_escape(diff_line.text) unless diff_file && diff_file.diff_refs line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' @@ -52,10 +56,12 @@ module Gitlab end def old_lines + return unless diff_file @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) end def new_lines + return unless diff_file @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) end -- cgit v1.2.1 From 82d0fa8e451de384150394d8465f6f165adfeae9 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 22 Jan 2016 01:53:01 +0100 Subject: Fix --- lib/gitlab/diff/highlight.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 964c89de6c6..9283b5e185d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -7,8 +7,8 @@ module Gitlab def initialize(diff_lines) if diff_lines.is_a?(Gitlab::Diff::File) - @diff_file = diff_file - @diff_lines = diff_file.diff_lines + @diff_file = diff_lines + @diff_lines = @diff_file.diff_lines else @diff_lines = diff_lines end -- cgit v1.2.1 From 16d17b78cabe2efd762a4aa0e9fa2c2792fab32d Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 22 Jan 2016 02:10:58 +0100 Subject: Restore diff comments --- lib/gitlab/diff/highlight.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 179f8164c84..ac8537ad31a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -12,9 +12,10 @@ module Gitlab end def highlight - @diff_lines.each_with_index do |diff_line, i| + @diff_lines.map.with_index do |diff_line, i| + diff_line = diff_line.dup # ignore highlighting for "match" lines - next if diff_line.type == 'match' || diff_line.type == 'nonewline' + next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' rich_line = highlight_line(diff_line, i) @@ -23,9 +24,9 @@ module Gitlab end diff_line.text = rich_line.html_safe - end - @diff_lines + diff_line + end end private -- cgit v1.2.1 From 3db24ec9e80369834ccf7d2423808fcf5e51b5a6 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 22 Jan 2016 02:12:43 +0100 Subject: Properly highlight right side of parallel diff --- lib/gitlab/diff/parallel_diff.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index c0db3559e3a..74f9b3c050a 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -11,7 +11,8 @@ module Gitlab lines = [] skip_next = false - diff_file.highlighted_diff_lines.each do |line| + highlighted_diff_lines = diff_file.highlighted_diff_lines + highlighted_diff_lines.each do |line| full_line = line.text type = line.type line_code = generate_line_code(diff_file.file_path, line) @@ -21,6 +22,7 @@ module Gitlab next_line = diff_file.next_line(line.index) if next_line + next_line = highlighted_diff_lines[next_line.index] next_line_code = generate_line_code(diff_file.file_path, next_line) next_type = next_line.type next_line = next_line.text -- cgit v1.2.1 From be7bc9d9b096a3d45bd8f58d1946bc80fea1c7f0 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Fri, 22 Jan 2016 11:24:38 +0200 Subject: Backport some changes from EE --- lib/gitlab/snippet_search_results.rb | 82 +----------------------------------- 1 file changed, 2 insertions(+), 80 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 938219efdb2..38364a0b151 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -1,5 +1,7 @@ module Gitlab class SnippetSearchResults < SearchResults + include SnippetsHelper + attr_reader :limit_snippet_ids def initialize(limit_snippet_ids, query) @@ -47,85 +49,5 @@ module Gitlab def default_scope 'snippet_blobs' end - - # Get an array of line numbers surrounding a matching - # line, bounded by min/max. - # - # @returns Array of line numbers - def bounded_line_numbers(line, min, max) - lower = line - surrounding_lines > min ? line - surrounding_lines : min - upper = line + surrounding_lines < max ? line + surrounding_lines : max - (lower..upper).to_a - end - - # Returns a sorted set of lines to be included in a snippet preview. - # This ensures matching adjacent lines do not display duplicated - # surrounding code. - # - # @returns Array, unique and sorted. - def matching_lines(lined_content) - used_lines = [] - lined_content.each_with_index do |line, line_number| - used_lines.concat bounded_line_numbers( - line_number, - 0, - lined_content.size - ) if line.include?(query) - end - - used_lines.uniq.sort - end - - # 'Chunkify' entire snippet. Splits the snippet data into matching lines + - # surrounding_lines() worth of unmatching lines. - # - # @returns a hash with {snippet_object, snippet_chunks:{data,start_line}} - def chunk_snippet(snippet) - lined_content = snippet.content.split("\n") - used_lines = matching_lines(lined_content) - - snippet_chunk = [] - snippet_chunks = [] - snippet_start_line = 0 - last_line = -1 - - # Go through each used line, and add consecutive lines as a single chunk - # to the snippet chunk array. - used_lines.each do |line_number| - if last_line < 0 - # Start a new chunk. - snippet_start_line = line_number - snippet_chunk << lined_content[line_number] - elsif last_line == line_number - 1 - # Consecutive line, continue chunk. - snippet_chunk << lined_content[line_number] - else - # Non-consecutive line, add chunk to chunk array. - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Start a new chunk. - snippet_chunk = [lined_content[line_number]] - snippet_start_line = line_number - end - last_line = line_number - end - # Add final chunk to chunk array - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Return snippet with chunk array - { snippet_object: snippet, snippet_chunks: snippet_chunks } - end - - # Defines how many unmatching lines should be - # included around the matching lines in a snippet - def surrounding_lines - 3 - end end end -- cgit v1.2.1 From 6435f78a8c66be92613c3a8ea4ec8171d0c38fea Mon Sep 17 00:00:00 2001 From: Benedict Etzel Date: Fri, 22 Jan 2016 13:47:14 +0100 Subject: Whitelist raw "abbr" elements when parsing Markdown Closes #12517 --- lib/banzai/filter/sanitization_filter.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 3f49d492f2f..d1e11eedec3 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -43,6 +43,10 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow abbr elements with title attribute + whitelist[:elements].push('abbr') + whitelist[:attributes]['abbr'] = %w(title) + # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') -- cgit v1.2.1 From cd22da3ebda969779c47f1652f3876daa5edd7f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 25 Jan 2016 20:00:59 +0100 Subject: Fix preventing migration from crashing in very specific cases See https://gitlab.com/gitlab-org/gitlab-ce/issues/12606 for details --- lib/gitlab/current_settings.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index ea054255820..429f1f9bb56 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -39,6 +39,13 @@ module Gitlab end use_db && ActiveRecord::Base.connection.active? && + # The following condition is important: if a migrations adds a + # column to the application_settings table and a validation in + # the ApplicationSetting uses this new column we might end-up in + # a vicious circle where migration crash before being done. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12606 for + # a thorough explanation. + !ActiveRecord::Migrator.needs_migration? && ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError -- cgit v1.2.1 From b74308c0a74ce9256bfe906a070b8751d2cc9e9e Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 25 Jan 2016 21:26:49 +0100 Subject: Correct arity for instrumented methods w/o args This ensures that an instrumented method that doesn't take arguments reports an arity of 0, instead of -1. If Ruby had a proper method for finding out the required arguments of a method (e.g. Method#required_arguments) this would not have been an issue. Sadly the only two methods we have are Method#parameters and Method#arity, and both are equally painful to use. Fixes gitlab-org/gitlab-ce#12450 --- lib/gitlab/metrics/instrumentation.rb | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d9fce2e6758..face1921d2e 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -106,20 +106,36 @@ module Gitlab if type == :instance target = mod label = "#{mod.name}##{name}" + method = mod.instance_method(name) else target = mod.singleton_class label = "#{mod.name}.#{name}" + method = mod.method(name) + end + + # Some code out there (e.g. the "state_machine" Gem) checks the arity of + # a method to make sure it only passes arguments when the method expects + # any. If we were to always overwrite a method to take an `*args` + # signature this would break things. As a result we'll make sure the + # generated method _only_ accepts regular arguments if the underlying + # method also accepts them. + if method.arity == 0 + args_signature = '&block' + else + args_signature = '*args, &block' end + send_signature = "__send__(#{alias_name.inspect}, #{args_signature})" + target.class_eval <<-EOF, __FILE__, __LINE__ + 1 alias_method #{alias_name.inspect}, #{name.inspect} - def #{name}(*args, &block) + def #{name}(#{args_signature}) trans = Gitlab::Metrics::Instrumentation.transaction if trans start = Time.now - retval = __send__(#{alias_name.inspect}, *args, &block) + retval = #{send_signature} duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold @@ -132,7 +148,7 @@ module Gitlab retval else - __send__(#{alias_name.inspect}, *args, &block) + #{send_signature} end end EOF -- cgit v1.2.1 From c0403234193dcb2033bd57160bb0ab6893bb8d77 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Jan 2016 16:53:34 -0200 Subject: Move Gitlab::BitbucketImport::KeyDeleter to it's own importer --- lib/gitlab/bitbucket_import/importer.rb | 47 ++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 2355b3c6ddc..e8c6d89764c 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -13,12 +13,34 @@ module Gitlab end def execute - project_identifier = project.import_source + import_issues if has_issues? - return true unless client.project(project_identifier)["has_issues"] + true + ensure + Gitlab::BitbucketImport::KeyDeleter.new(project).execute + end - #Issues && Comments - issues = client.issues(project_identifier) + private + + def gl_user_id(project, bitbucket_id) + if bitbucket_id + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + else + project.creator_id + end + end + + def identifier + project.import_source + end + + def has_issues? + client.project(identifier)["has_issues"] + end + + def import_issues + issues = client.issues(identifier) issues.each do |issue| body = '' @@ -33,7 +55,7 @@ module Gitlab body = @formatter.author_line(author) body += issue["content"] - comments = client.issue_comments(project_identifier, issue["local_id"]) + comments = client.issue_comments(identifier, issue["local_id"]) if comments.any? body += @formatter.comments_header @@ -56,20 +78,9 @@ module Gitlab author_id: gl_user_id(project, reporter) ) end - - true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end - - private - - def gl_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end - end end end end -- cgit v1.2.1 From b58a2e30b214ffc98f492aab88137cc3fd48355d Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 22 Jan 2016 13:00:00 -0200 Subject: Wrap errors on GitHub importer to raise Projects::ImportService::Error --- lib/gitlab/bitbucket_import/importer.rb | 2 ++ lib/gitlab/github_import/importer.rb | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e8c6d89764c..3f483847efa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -16,6 +16,8 @@ module Gitlab import_issues if has_issues? true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error.new, e.message ensure Gitlab::BitbucketImport::KeyDeleter.new(project).execute end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 663402e8197..e2a85f29825 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -35,8 +35,8 @@ module Gitlab end true - rescue ActiveRecord::RecordInvalid - false + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end def import_pull_requests @@ -53,8 +53,8 @@ module Gitlab end true - rescue ActiveRecord::RecordInvalid - false + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end def import_comments(issue_number, noteable) @@ -83,10 +83,13 @@ module Gitlab true rescue Gitlab::Shell::Error => e - if e.message =~ /repository not exported/ - true + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + raise Projects::ImportService::Error, e.message else - false + true end end end -- cgit v1.2.1 From e6f3fe5d3b16809634dd655b3d265c683b538b12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 26 Jan 2016 15:11:15 +0100 Subject: Ensure rake tasks that don't need a DB connection can be run without one When using ActiveRecord::Base.connection.active? without a DB connection, we get a "PG::ConnectionBad: could not connect to server" error because ActiveRecord::Base.connection doesn't exist. By using ActiveRecord::Base.connected? we ensure we don't get this error if the connection is missing, which is the all point of the Gitlab::CurrentSettings#connect_to_db? method! --- lib/gitlab/current_settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 429f1f9bb56..19b7427256c 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -38,7 +38,7 @@ module Gitlab true end - use_db && ActiveRecord::Base.connection.active? && + use_db && ActiveRecord::Base.connected? && # The following condition is important: if a migrations adds a # column to the application_settings table and a validation in # the ApplicationSetting uses this new column we might end-up in -- cgit v1.2.1 From 5ea32ba4982aef37f2a43b0324d45033992a9a60 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 26 Jan 2016 19:05:04 +0100 Subject: Fix hot reloading for CI API --- lib/ci/api/api.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 5c347e432b4..4e85d2c3c74 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -25,7 +25,7 @@ module Ci format :json - helpers Helpers + helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings -- cgit v1.2.1 From 95d2f0fb51de2c5800d41b7c4f285ed533ffe21a Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 26 Jan 2016 19:25:48 +0100 Subject: Fix CI runner version not being properly updated when asking for a build Due to broken implementation of attribute_for_keys the runner information was not updated correctly. This MR adds test to check that such scenario will never happen again. --- lib/api/helpers.rb | 5 +++-- lib/ci/api/builds.rb | 2 +- lib/ci/api/helpers.rb | 10 +++++++--- lib/ci/api/runners.rb | 1 + 4 files changed, 12 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3f528b9f7c0..9dacf7c1e86 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -153,10 +153,11 @@ module API end def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) - attrs[key] = params[key] + if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + attrs[key] = params_hash[key] end end ActionController::Parameters.new(attrs).permit! diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 690bbf97a89..416b0b5f0b4 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -13,13 +13,13 @@ module Ci post "register" do authenticate_runner! update_runner_last_contact + update_runner_info required_attributes! [:token] not_found! unless current_runner.active? build = Ci::RegisterBuildService.new.execute(current_runner) if build - update_runner_info present build, with: Entities::BuildDetails else not_found! diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 1c91204e98c..199d62d9b8a 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -34,10 +34,14 @@ module Ci @runner ||= Runner.find_by_token(params[:token].to_s) end - def update_runner_info + def get_runner_version_from_params return unless params["info"].present? - info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) - current_runner.update(info) + attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) + end + + def update_runner_info + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? end def max_artifacts_size diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index bfc14fe7a6b..192b1d18a51 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -47,6 +47,7 @@ module Ci return forbidden! unless runner if runner.id + runner.update(get_runner_version_from_params) present runner, with: Entities::Runner else not_found! -- cgit v1.2.1 From 869b4d7c6ad5c025f310eeecce7293bf5bbc4926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 28 Jan 2016 10:14:21 +0100 Subject: Only create the defaults ApplicationSetting when DB is fully migrated Return a fake application settings OpenStruct when this is not the case. Also, use ActiveRecord::Base.connection_pool.active_connection? instead of ActiveRecord::Base.connection.active? to avoid driver exception. --- lib/gitlab/current_settings.rb | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 19b7427256c..e66e7768f2b 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -4,11 +4,14 @@ module Gitlab key = :current_application_settings RequestStore.store[key] ||= begin + settings = nil + if connect_to_db? - ApplicationSetting.current || ApplicationSetting.create_from_defaults - else - fake_application_settings + settings = ApplicationSetting.current + settings ||= ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end end @@ -18,35 +21,29 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], + twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], sign_in_text: Settings.extra['sign_in_text'], restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'], + default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], + default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], + import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], + require_two_factor_authentication: false, + two_factor_grace_period: 48 ) end private def connect_to_db? - use_db = if ENV['USE_DB'] == "false" - false - else - true - end - - use_db && ActiveRecord::Base.connected? && - # The following condition is important: if a migrations adds a - # column to the application_settings table and a validation in - # the ApplicationSetting uses this new column we might end-up in - # a vicious circle where migration crash before being done. - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12606 for - # a thorough explanation. - !ActiveRecord::Migrator.needs_migration? && - ActiveRecord::Base.connection.table_exists?('application_settings') + ENV['USE_DB'] != 'false' && + ActiveRecord::Base.connection_pool.active_connection? && + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false -- cgit v1.2.1 From 55ab92c00bb79a951dea477d59e440c80dc96f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 28 Jan 2016 12:23:37 +0100 Subject: Use ActiveRecord::Base.connection.active? and rescue any exception in connect_to_db? This ensures that rake tasks that don't need a DB connection can be run without one. --- lib/gitlab/current_settings.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index e66e7768f2b..a6b2f14521c 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -41,8 +41,11 @@ module Gitlab private def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false + ENV['USE_DB'] != 'false' && - ActiveRecord::Base.connection_pool.active_connection? && + active_db_connection && ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError -- cgit v1.2.1 From dfb8803c1cf5946d3eb7d0e70ba6fa8896f67a6f Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 28 Jan 2016 14:26:02 +0100 Subject: Improve consistency and duplication for Merge Request API * Follow REST for merge request API route * Remove repeating comments API for MR Signed-off-by: Dmitriy Zaporozhets --- lib/api/merge_requests.rb | 351 ++++++++++++++++++++++++---------------------- 1 file changed, 181 insertions(+), 170 deletions(-) (limited to 'lib') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 5c97fe1c88c..dd7f24f3279 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -59,55 +59,6 @@ module API present paginate(merge_requests), with: Entities::MergeRequest end - # Show MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id - # - get ":id/merge_request/:merge_request_id" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - - present merge_request, with: Entities::MergeRequest - end - - # Show MR commits - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id/commits - # - get ':id/merge_request/:merge_request_id/commits' do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request.commits, with: Entities::RepoCommit - end - - # Show MR changes - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id/changes - # - get ':id/merge_request/:merge_request_id/changes' do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges - end - # Create MR # # Parameters: @@ -148,146 +99,206 @@ module API end end - # Update MR + # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 + # Use "merge_requests/:merge_request_id/..." instead. # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # target_branch - The target branch - # assignee_id - Assignee user ID - # title - Title of MR - # state_event - Status of MR. (close|reopen|merge) - # description - Description of MR - # labels (optional) - Labels for a MR as a comma-separated list - # Example: - # PUT /projects/:id/merge_request/:merge_request_id - # - put ":id/merge_request/:merge_request_id" do - attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :update_merge_request, merge_request - - # Ensure source_branch is not specified - if params[:source_branch].present? - render_api_error!('Source branch cannot be changed', 400) - end - - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end - - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) - - if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end + [":id/merge_request/:merge_request_id", ":id/merge_requests/:merge_request_id"].each do |path| + # Show MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id + # + get path do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request present merge_request, with: Entities::MergeRequest - else - handle_merge_request_errors! merge_request.errors end - end - - # Merge MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # merge_commit_message (optional) - Custom merge commit message - # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible - # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds - # Example: - # PUT /projects/:id/merge_request/:merge_request_id/merge - # - put ":id/merge_request/:merge_request_id/merge" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged - - render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? - - merge_params = { - commit_message: params[:merge_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] - } - - if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? - ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) - else - ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + # Show MR commits + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id/commits + # + get "#{path}/commits" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request.commits, with: Entities::RepoCommit end - present merge_request, with: Entities::MergeRequest - end + # Show MR changes + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id/changes + # + get "#{path}/changes" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request, with: Entities::MergeRequestChanges + end - # Cancel Merge if Merge When build succeeds is enabled - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # - post ":id/merge_request/:merge_request_id/cancel_merge_when_build_succeeds" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + # Update MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # target_branch - The target branch + # assignee_id - Assignee user ID + # title - Title of MR + # state_event - Status of MR. (close|reopen|merge) + # description - Description of MR + # labels (optional) - Labels for a MR as a comma-separated list + # Example: + # PUT /projects/:id/merge_requests/:merge_request_id + # + put path do + attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description] + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + authorize! :update_merge_request, merge_request + + # Ensure source_branch is not specified + if params[:source_branch].present? + render_api_error!('Source branch cannot be changed', 400) + end - unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + # Validate label names in advance + if (errors = validate_label_params(params)).any? + render_api_error!({ labels: errors }, 400) + end - ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) - end + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) - # Get a merge request's comments - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_request/:merge_request_id/comments - # - get ":id/merge_request/:merge_request_id/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + if merge_request.valid? + # Find or create labels and attach to issue + unless params[:labels].nil? + merge_request.remove_labels + merge_request.add_labels_by_names(params[:labels].split(",")) + end - authorize! :read_merge_request, merge_request + present merge_request, with: Entities::MergeRequest + else + handle_merge_request_errors! merge_request.errors + end + end - present paginate(merge_request.notes.fresh), with: Entities::MRNote - end + # Merge MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # merge_commit_message (optional) - Custom merge commit message + # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible + # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds + # Example: + # PUT /projects/:id/merge_requests/:merge_request_id/merge + # + put "#{path}/merge" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) + not_allowed! if !merge_request.open? || merge_request.work_in_progress? + + merge_request.check_if_can_be_merged + + render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? + + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? + ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) + else + ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) + end - # Post comment to merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # note (required) - Text of comment - # Examples: - # POST /projects/:id/merge_request/:merge_request_id/comments - # - post ":id/merge_request/:merge_request_id/comments" do - required_attributes! [:note] + present merge_request, with: Entities::MergeRequest + end - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + # Cancel Merge if Merge When build succeeds is enabled + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # + post "#{path}/cancel_merge_when_build_succeeds" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :create_note, merge_request + unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) - opts = { - note: params[:note], - noteable_type: 'MergeRequest', - noteable_id: merge_request.id - } + ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) + end - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. + # Use GET "/projects/:id/merge_requests/:merge_request_id/notes" instead + # + # Get a merge request's comments + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # Examples: + # GET /projects/:id/merge_requests/:merge_request_id/comments + # + get "#{path}/comments" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + + present paginate(merge_request.notes.fresh), with: Entities::MRNote + end - if note.save - present note, with: Entities::MRNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) + # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. + # Use POST "/projects/:id/merge_requests/:merge_request_id/notes" instead + # + # Post comment to merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # note (required) - Text of comment + # Examples: + # POST /projects/:id/merge_requests/:merge_request_id/comments + # + post "#{path}/comments" do + required_attributes! [:note] + + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: Entities::MRNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end end end end -- cgit v1.2.1 From b636f83e5910d10a954464370efe42cc14a097ba Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 28 Jan 2016 14:49:21 +0100 Subject: Move blame group logic to dedicated class --- lib/gitlab/blame.rb | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 lib/gitlab/blame.rb (limited to 'lib') diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb new file mode 100644 index 00000000000..313e6b9fc03 --- /dev/null +++ b/lib/gitlab/blame.rb @@ -0,0 +1,54 @@ +module Gitlab + class Blame + attr_accessor :blob, :commit + + def initialize(blob, commit) + @blob = blob + @commit = commit + end + + def groups(highlight: true) + prev_sha = nil + groups = [] + current_group = nil + + i = 0 + blame.each do |commit, line| + commit = Commit.new(commit, project) + + sha = commit.sha + if prev_sha != sha + groups << current_group if current_group + current_group = { commit: commit, lines: [] } + end + + line = highlighted_lines[i].html_safe if highlight + current_group[:lines] << line + + prev_sha = sha + i += 1 + end + groups << current_group if current_group + + groups + end + + private + + def blame + @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path) + end + + def highlighted_lines + @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + end + + def project + commit.project + end + + def repository + project.repository + end + end +end -- cgit v1.2.1 From ca05054ea2a8aff81822f310c5dafb68ae26e10e Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 28 Jan 2016 16:28:19 -0500 Subject: Partially revert "Add IP check against DNSBLs at account sign-up" This partially reverts 6a5cd3ca - we keep the migration and add a new migration that reverts it in order to keep migration history intact. --- lib/dnsxl_check.rb | 105 ------------------------------------------------- lib/gitlab/ip_check.rb | 34 ---------------- 2 files changed, 139 deletions(-) delete mode 100644 lib/dnsxl_check.rb delete mode 100644 lib/gitlab/ip_check.rb (limited to 'lib') diff --git a/lib/dnsxl_check.rb b/lib/dnsxl_check.rb deleted file mode 100644 index 1e506b2d9cb..00000000000 --- a/lib/dnsxl_check.rb +++ /dev/null @@ -1,105 +0,0 @@ -require 'resolv' - -class DNSXLCheck - - class Resolver - def self.search(query) - begin - Resolv.getaddress(query) - true - rescue Resolv::ResolvError - false - end - end - end - - IP_REGEXP = /\A(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\z/ - DEFAULT_THRESHOLD = 0.33 - - def self.create_from_list(list) - dnsxl_check = DNSXLCheck.new - - list.each do |entry| - dnsxl_check.add_list(entry.domain, entry.weight) - end - - dnsxl_check - end - - def test(ip) - if use_threshold? - test_with_threshold(ip) - else - test_strict(ip) - end - end - - def test_with_threshold(ip) - return false if lists.empty? - - search(ip) - final_score >= threshold - end - - def test_strict(ip) - return false if lists.empty? - - search(ip) - @score > 0 - end - - def use_threshold=(value) - @use_threshold = value == true - end - - def use_threshold? - @use_threshold &&= true - end - - def threshold=(threshold) - raise ArgumentError, "'threshold' value must be grather than 0 and less than or equal to 1" unless threshold > 0 && threshold <= 1 - @threshold = threshold - end - - def threshold - @threshold ||= DEFAULT_THRESHOLD - end - - def add_list(domain, weight) - @lists ||= [] - @lists << { domain: domain, weight: weight } - end - - def lists - @lists ||= [] - end - - private - - def search(ip) - raise ArgumentError, "'ip' value must be in #{IP_REGEXP} format" unless ip.match(IP_REGEXP) - - @score = 0 - - reversed = reverse_ip(ip) - search_in_rbls(reversed) - end - - def reverse_ip(ip) - ip.split('.').reverse.join('.') - end - - def search_in_rbls(reversed_ip) - lists.each do |rbl| - query = "#{reversed_ip}.#{rbl[:domain]}" - @score += rbl[:weight] if Resolver.search(query) - end - end - - def final_score - weights = lists.map{ |rbl| rbl[:weight] }.reduce(:+).to_i - return 0 if weights == 0 - - (@score.to_f / weights.to_f).round(2) - end -end diff --git a/lib/gitlab/ip_check.rb b/lib/gitlab/ip_check.rb deleted file mode 100644 index f2e9b50d225..00000000000 --- a/lib/gitlab/ip_check.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - class IpCheck - - def initialize(ip) - @ip = ip - - application_settings = ApplicationSetting.current - @ip_blocking_enabled = application_settings.ip_blocking_enabled - @dnsbl_servers_list = application_settings.dnsbl_servers_list - end - - def spam? - @ip_blocking_enabled && blacklisted? - end - - private - - def blacklisted? - on_dns_blacklist? - end - - def on_dns_blacklist? - dnsbl_check = DNSXLCheck.new - prepare_dnsbl_list(dnsbl_check) - dnsbl_check.test(@ip) - end - - def prepare_dnsbl_list(dnsbl_check) - @dnsbl_servers_list.split(',').map(&:strip).reject(&:empty?).each do |domain| - dnsbl_check.add_list(domain, 1) - end - end - end -end -- cgit v1.2.1 From 91b9cbff8de538c2966bca94c517ee6aa346f79b Mon Sep 17 00:00:00 2001 From: Josh Frye Date: Fri, 22 Jan 2016 11:13:37 -0800 Subject: First pass at deleting projects in the background. --- lib/api/projects.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 71bb342f844..1f991e600e3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -187,7 +187,7 @@ module API else present @forked_project, with: Entities::Project, user_can_admin_project: can?(current_user, :admin_project, @forked_project) - end + end end # Update an existing project @@ -246,7 +246,7 @@ module API # DELETE /projects/:id delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).execute + ::Projects::DestroyService.new(user_project, current_user, {}).pending_delete! end # Mark this project as forked from another -- cgit v1.2.1 From 677b4db9e682b29bb15dddb84fe80b976490a671 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 29 Jan 2016 19:37:17 +0100 Subject: Mark inline difference between old and new paths when a file is renamed --- lib/gitlab/diff/highlight.rb | 17 ++++------- lib/gitlab/diff/inline_diff.rb | 57 ++++++++++++++++++++++------------- lib/gitlab/diff/inline_diff_marker.rb | 14 ++++++--- 3 files changed, 52 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a7f925ce134..9429b3ff88d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -21,13 +21,13 @@ module Gitlab # ignore highlighting for "match" lines next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' - rich_line = highlight_line(diff_line, i) + rich_line = highlight_line(diff_line) || diff_line.text if line_inline_diffs = inline_diffs[i] rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) end - diff_line.text = rich_line.html_safe + diff_line.text = rich_line diff_line end @@ -35,8 +35,8 @@ module Gitlab private - def highlight_line(diff_line, index) - return html_escape(diff_line.text) unless diff_file && diff_file.diff_refs + def highlight_line(diff_line) + return unless diff_file && diff_file.diff_refs line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' @@ -49,11 +49,11 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - rich_line ? line_prefix + rich_line : html_escape(diff_line.text) + "#{line_prefix}#{rich_line}".html_safe if rich_line end def inline_diffs - @inline_diffs ||= InlineDiff.new(@raw_lines).inline_diffs + @inline_diffs ||= InlineDiff.for_lines(@raw_lines) end def old_lines @@ -72,11 +72,6 @@ module Gitlab [ref.project.repository, ref.id, path] end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index b8a61ad6115..789c14518b0 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,43 +1,58 @@ module Gitlab module Diff class InlineDiff - attr_accessor :lines + attr_accessor :old_line, :new_line, :offset - def initialize(lines) - @lines = lines - end + def self.for_lines(lines) + local_edit_indexes = self.find_local_edits(lines) - def inline_diffs inline_diffs = [] local_edit_indexes.each do |index| old_index = index new_index = index + 1 - old_line = @lines[old_index] - new_line = @lines[new_index] - - # Skip inline diff if empty line was replaced with content - next if old_line[1..-1] == "" - - # Add one, because this is based on the prefixless version - lcp = longest_common_prefix(old_line[1..-1], new_line[1..-1]) + 1 - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + old_line = lines[old_index] + new_line = lines[new_index] - old_diff_range = lcp..(old_line.length - lcs - 1) - new_diff_range = lcp..(new_line.length - lcs - 1) + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs - inline_diffs[old_index] = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - inline_diffs[new_index] = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + inline_diffs[old_index] = old_diffs + inline_diffs[new_index] = new_diffs end inline_diffs end + def initialize(old_line, new_line, offset: 0) + @old_line = old_line[offset..-1] + @new_line = new_line[offset..-1] + @offset = offset + end + + def inline_diffs + # Skip inline diff if empty line was replaced with content + return if old_line == "" + + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + private - # Find runs of single line edits - def local_edit_indexes - line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + def self.find_local_edits(lines) + line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } joined_line_prefixes = " #{line_prefixes.join} " offset = 0 diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 1d7fa1bce06..dccb717e95d 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -5,10 +5,12 @@ module Gitlab def initialize(raw_line, rich_line = raw_line) @raw_line = raw_line - @rich_line = rich_line + @rich_line = ERB::Util.html_escape(rich_line) end def mark(line_inline_diffs) + return rich_line unless line_inline_diffs + marker_ranges = [] line_inline_diffs.each do |inline_diff_range| # Map the inline-diff range based on the raw line to character positions in the rich line @@ -19,11 +21,15 @@ module Gitlab offset = 0 # Mark each range - marker_ranges.each do |range| - offset = insert_around_range(rich_line, range, "", "", offset) + marker_ranges.each_with_index do |range, i| + class_names = ["idiff"] + class_names << "left" if i == 0 + class_names << "right" if i == marker_ranges.length - 1 + + offset = insert_around_range(rich_line, range, "", "", offset) end - rich_line + rich_line.html_safe end private -- cgit v1.2.1