diff options
Diffstat (limited to 'lib')
24 files changed, 671 insertions, 152 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6d2380cf47d..3f528b9f7c0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,6 +264,10 @@ module API projects = projects.search(params[:search]) end + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) + end + projects.reorder(project_order_by => project_sort) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 20bd4f7ee6e..3637b1bac94 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -133,6 +133,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -170,6 +171,7 @@ module Banzai next unless link && text link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? next unless link && link =~ /\A#{pattern}\z/ html = yield link, text diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index bcdfd38d292..1a3f662811a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -115,6 +115,10 @@ module Ci end if @cache + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" + end + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) raise ValidationError, "cache:untracked parameter should be an boolean" end @@ -198,6 +202,10 @@ module Ci end def validate_job_cache!(name, job) + if job[:cache][:key] && !validate_string(job[:cache][:key]) + raise ValidationError, "#{name} job: cache:key parameter should be a string" + end + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" end diff --git a/lib/dnsxl_check.rb b/lib/dnsxl_check.rb new file mode 100644 index 00000000000..1e506b2d9cb --- /dev/null +++ b/lib/dnsxl_check.rb @@ -0,0 +1,105 @@ +require 'resolv' + +class DNSXLCheck + + class Resolver + def self.search(query) + begin + Resolv.getaddress(query) + true + rescue Resolv::ResolvError + false + end + end + end + + IP_REGEXP = /\A(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\z/ + DEFAULT_THRESHOLD = 0.33 + + def self.create_from_list(list) + dnsxl_check = DNSXLCheck.new + + list.each do |entry| + dnsxl_check.add_list(entry.domain, entry.weight) + end + + dnsxl_check + end + + def test(ip) + if use_threshold? + test_with_threshold(ip) + else + test_strict(ip) + end + end + + def test_with_threshold(ip) + return false if lists.empty? + + search(ip) + final_score >= threshold + end + + def test_strict(ip) + return false if lists.empty? + + search(ip) + @score > 0 + end + + def use_threshold=(value) + @use_threshold = value == true + end + + def use_threshold? + @use_threshold &&= true + end + + def threshold=(threshold) + raise ArgumentError, "'threshold' value must be grather than 0 and less than or equal to 1" unless threshold > 0 && threshold <= 1 + @threshold = threshold + end + + def threshold + @threshold ||= DEFAULT_THRESHOLD + end + + def add_list(domain, weight) + @lists ||= [] + @lists << { domain: domain, weight: weight } + end + + def lists + @lists ||= [] + end + + private + + def search(ip) + raise ArgumentError, "'ip' value must be in #{IP_REGEXP} format" unless ip.match(IP_REGEXP) + + @score = 0 + + reversed = reverse_ip(ip) + search_in_rbls(reversed) + end + + def reverse_ip(ip) + ip.split('.').reverse.join('.') + end + + def search_in_rbls(reversed_ip) + lists.each do |rbl| + query = "#{reversed_ip}.#{rbl[:domain]}" + @score += rbl[:weight] if Resolver.search(query) + end + end + + def final_score + weights = lists.map{ |rbl| rbl[:weight] }.reduce(:+).to_i + return 0 if weights == 0 + + (@score.to_f / weights.to_f).round(2) + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1344f5d120b..f2020c82d40 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -13,8 +13,8 @@ module Gitlab attr_reader :file, :path, :full_version - def initialize(file, path) - @file, @path = file, path + def initialize(file, path, **opts) + @file, @path, @opts = file, path, opts @full_version = read_version end @@ -52,7 +52,9 @@ module Gitlab def match_entries(gz) entries = {} - match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} + + child_pattern = '[^/]*/?$' unless @opts[:recursive] + match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ until gz.eof? do begin diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 25b71fc3275..7f4c750b6fd 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -95,6 +95,13 @@ module Gitlab children.empty? end + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + def to_s @path end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 79061cd0141..a484177ae33 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,13 +1,22 @@ module Gitlab module Diff class File - attr_reader :diff + attr_reader :diff, :diff_refs delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff) + def initialize(diff, diff_refs) @diff = diff + @diff_refs = diff_refs + end + + def old_ref + diff_refs[0] if diff_refs + end + + def new_ref + diff_refs[1] if diff_refs end # Array of Gitlab::DIff::Line objects @@ -15,6 +24,14 @@ module Gitlab @lines ||= parser.parse(raw_diff.lines) end + def highlighted_diff_lines + Gitlab::Diff::Highlight.new(self).highlight + end + + def parallel_diff_lines + Gitlab::Diff::ParallelDiff.new(self).parallelize + end + def mode_changed? !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb new file mode 100644 index 00000000000..a7f925ce134 --- /dev/null +++ b/lib/gitlab/diff/highlight.rb @@ -0,0 +1,82 @@ +module Gitlab + module Diff + class Highlight + attr_reader :diff_file, :diff_lines, :raw_lines + + delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + + def initialize(diff_lines) + if diff_lines.is_a?(Gitlab::Diff::File) + @diff_file = diff_lines + @diff_lines = @diff_file.diff_lines + else + @diff_lines = diff_lines + end + @raw_lines = @diff_lines.map(&:text) + end + + def highlight + @diff_lines.map.with_index do |diff_line, i| + diff_line = diff_line.dup + # ignore highlighting for "match" lines + next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + + rich_line = highlight_line(diff_line, i) + + if line_inline_diffs = inline_diffs[i] + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + end + + diff_line.text = rich_line.html_safe + + diff_line + end + end + + private + + def highlight_line(diff_line, index) + return html_escape(diff_line.text) unless diff_file && diff_file.diff_refs + + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + + case diff_line.type + when 'new', nil + rich_line = new_lines[diff_line.new_pos - 1] + when 'old' + rich_line = old_lines[diff_line.old_pos - 1] + end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + rich_line ? line_prefix + rich_line : html_escape(diff_line.text) + end + + def inline_diffs + @inline_diffs ||= InlineDiff.new(@raw_lines).inline_diffs + end + + def old_lines + return unless diff_file + @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + end + + def new_lines + return unless diff_file + @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) + end + + def processing_args(version) + ref = send("diff_#{version}_ref") + path = send("diff_#{version}_path") + + [ref.project.repository, ref.id, path] + end + + def html_escape(str) + replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } + str.gsub(/[&"'><]/, replacements) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb new file mode 100644 index 00000000000..b8a61ad6115 --- /dev/null +++ b/lib/gitlab/diff/inline_diff.rb @@ -0,0 +1,73 @@ +module Gitlab + module Diff + class InlineDiff + attr_accessor :lines + + def initialize(lines) + @lines = lines + end + + def inline_diffs + inline_diffs = [] + + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = @lines[old_index] + new_line = @lines[new_index] + + # Skip inline diff if empty line was replaced with content + next if old_line[1..-1] == "" + + # Add one, because this is based on the prefixless version + lcp = longest_common_prefix(old_line[1..-1], new_line[1..-1]) + 1 + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + old_diff_range = lcp..(old_line.length - lcs - 1) + new_diff_range = lcp..(new_line.length - lcs - 1) + + inline_diffs[old_index] = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + inline_diffs[new_index] = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + end + + inline_diffs + end + + private + + # Find runs of single line edits + def local_edit_indexes + line_prefixes = @lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 + end + + local_edit_indexes + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb new file mode 100644 index 00000000000..1d7fa1bce06 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -0,0 +1,109 @@ +module Gitlab + module Diff + class InlineDiffMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = rich_line + end + + def mark(line_inline_diffs) + marker_ranges = [] + line_inline_diffs.each do |inline_diff_range| + # Map the inline-diff range based on the raw line to character positions in the rich line + inline_diff_positions = position_mapping[inline_diff_range].flatten + # Turn the array of character positions into ranges + marker_ranges.concat(collapse_ranges(inline_diff_positions)) + end + + offset = 0 + # Mark each range + marker_ranges.each do |range| + offset = insert_around_range(rich_line, range, "<span class='idiff'>", "</span>", offset) + end + + rich_line + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + # Inserts tags around the characters identified by the given range + def insert_around_range(text, range, before, after, offset = 0) + # Just to be sure + return offset if offset + range.end + 1 > text.length + + text.insert(offset + range.begin, before) + offset += before.length + + text.insert(offset + range.end + 1, after) + offset += after.length + + offset + end + end + end +end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0072194606e..03730b435ad 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,7 +1,8 @@ module Gitlab module Diff class Line - attr_reader :type, :text, :index, :old_pos, :new_pos + attr_reader :type, :index, :old_pos, :new_pos + attr_accessor :text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb new file mode 100644 index 00000000000..74f9b3c050a --- /dev/null +++ b/lib/gitlab/diff/parallel_diff.rb @@ -0,0 +1,119 @@ +module Gitlab + module Diff + class ParallelDiff + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + def parallelize + lines = [] + skip_next = false + + highlighted_diff_lines = diff_file.highlighted_diff_lines + highlighted_diff_lines.each do |line| + full_line = line.text + type = line.type + line_code = generate_line_code(diff_file.file_path, line) + line_new = line.new_pos + line_old = line.old_pos + + next_line = diff_file.next_line(line.index) + + if next_line + next_line = highlighted_diff_lines[next_line.index] + next_line_code = generate_line_code(diff_file.file_path, next_line) + next_type = next_line.type + next_line = next_line.text + end + + case type + when 'match', nil + # line in the right panel is the same as in the left one + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + when 'old' + case next_type + when 'new' + # Left side has text removed, right side has text added + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: line_new, + text: next_line, + line_code: next_line_code + } + } + skip_next = true + when 'old', 'nonewline', nil + # Left side has text removed, right side doesn't have any change + # No next line code, no new line number, no new line text + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: nil, + text: "", + line_code: nil + } + } + end + when 'new' + if skip_next + # Change has been already included in previous line so no need to do it again + skip_next = false + next + else + # Change is only on the right side, left side has no change + lines << { + left: { + type: nil, + number: nil, + text: "", + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + end + end + end + lines + end + + private + + def generate_line_code(file_path, line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 516e59b87a3..3666063bf8b 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -11,13 +11,10 @@ module Gitlab line_new = 1 type = nil - lines_arr = ::Gitlab::InlineDiff.processing lines - - lines_arr.each do |line| + @lines.each do |line| next if filename?(line) - full_line = html_escape(line.gsub(/\n/, '')) - full_line = ::Gitlab::InlineDiff.replace_markers full_line + full_line = line.gsub(/\n/, '') if line.match(/^@@ -/) type = "match" @@ -29,6 +26,10 @@ module Gitlab lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next + elsif line[0] == '\\' + type = 'nonewline' + lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 else type = identification_type(line) lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) @@ -36,10 +37,13 @@ module Gitlab end - if line[0] == "+" + case line[0] + when "+" line_new += 1 - elsif line[0] == "-" + when "-" line_old += 1 + when "\\" + # No increment else line_new += 1 line_old += 1 @@ -62,19 +66,15 @@ module Gitlab end def identification_type(line) - if line[0] == "+" + case line[0] + when "+" "new" - elsif line[0] == "-" + when "-" "old" else nil end end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index a2eb7a70bd2..a05ffeb9cd2 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -9,6 +9,7 @@ module Gitlab delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author + delegate :username, to: :author, prefix: :author def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 18929b9113b..663402e8197 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -82,8 +82,12 @@ module Gitlab end true - rescue Gitlab::Shell::Error - false + rescue Gitlab::Shell::Error => e + if e.message =~ /repository not exported/ + true + else + false + end end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b7c47958cc7..f96fed0f5cf 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -18,7 +18,7 @@ module Gitlab end def cross_project? - source_repo.fork == true + source_repo.id != target_repo.id end def number @@ -73,6 +73,10 @@ module Gitlab project end + def target_repo + raw_data.base.repo + end + def target_branch target_project.repository.find_branch(raw_data.base.ref) end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb new file mode 100644 index 00000000000..4ddb4fea977 --- /dev/null +++ b/lib/gitlab/highlight.rb @@ -0,0 +1,38 @@ +module Gitlab + class Highlight + def self.highlight(blob_name, blob_content, nowrap: true) + new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) + end + + def self.highlight_lines(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + highlight(file_name, blob.data).lines.map!(&:html_safe) + end + + def initialize(blob_name, blob_content, nowrap: true) + @formatter = rouge_formatter(nowrap: nowrap) + @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + end + + def highlight(text, continue: true) + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + rescue + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + private + + def rouge_formatter(options = {}) + options = options.reverse_merge( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + + Rouge::Formatters::HTMLGitlab.new(options) + end + end +end diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb deleted file mode 100644 index 44507bde25d..00000000000 --- a/lib/gitlab/inline_diff.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Gitlab - class InlineDiff - class << self - - START = "#!idiff-start!#" - FINISH = "#!idiff-finish!#" - - def processing(diff_arr) - indexes = _indexes_of_changed_lines diff_arr - - indexes.each do |index| - first_line = diff_arr[index+1] - second_line = diff_arr[index+2] - - # Skip inline diff if empty line was replaced with content - next if first_line == "-\n" - - first_token = find_first_token(first_line, second_line) - apply_first_token(diff_arr, index, first_token) - - last_token = find_last_token(first_line, second_line, first_token) - apply_last_token(diff_arr, index, last_token) - end - - diff_arr - end - - def apply_first_token(diff_arr, index, first_token) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end - end - - def apply_last_token(diff_arr, index, last_token) - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - end - - def find_first_token(first_line, second_line) - max_length = [first_line.size, second_line.size].max - first_the_same_symbols = 0 - - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_line[0..first_the_same_symbols][1..-1] - end - - def find_last_token(first_line, second_line, first_token) - max_length = [first_line.size, second_line.size].max - last_the_same_symbols = 0 - - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - - if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - - last_the_same_symbols += 1 - first_line[last_the_same_symbols..-1] - end - - def _indexes_of_changed_lines(diff_arr) - chain_of_first_symbols = "" - diff_arr.each_with_index do |line, i| - chain_of_first_symbols += line[0] - end - chain_of_first_symbols.gsub!(/[^\-\+]/, "#") - - offset = 0 - indexes = [] - while index = chain_of_first_symbols.index("#-+#", offset) - indexes << index - offset = index + 1 - end - indexes - end - - def replace_markers(line) - line.gsub!(START, "<span class='idiff'>") - line.gsub!(FINISH, "</span>") - line - end - end - end -end diff --git a/lib/gitlab/ip_check.rb b/lib/gitlab/ip_check.rb new file mode 100644 index 00000000000..f2e9b50d225 --- /dev/null +++ b/lib/gitlab/ip_check.rb @@ -0,0 +1,34 @@ +module Gitlab + class IpCheck + + def initialize(ip) + @ip = ip + + application_settings = ApplicationSetting.current + @ip_blocking_enabled = application_settings.ip_blocking_enabled + @dnsbl_servers_list = application_settings.dnsbl_servers_list + end + + def spam? + @ip_blocking_enabled && blacklisted? + end + + private + + def blacklisted? + on_dns_blacklist? + end + + def on_dns_blacklist? + dnsbl_check = DNSXLCheck.new + prepare_dnsbl_list(dnsbl_check) + dnsbl_check.test(@ip) + end + + def prepare_dnsbl_list(dnsbl_check) + @dnsbl_servers_list.split(',').map(&:strip).reject(&:empty?).each do |domain| + dnsbl_check.add_list(domain, 1) + end + end + end +end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index aef08c97d1d..e044f0ecc6d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -30,28 +30,31 @@ module Gitlab end def find_by_uid_and_provider - self.class.find_by_uid_and_provider( - auth_hash.uid, auth_hash.provider) + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? end def update_user_attributes - return unless persisted? + if persisted? + if auth_hash.has_email? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) + # For a new identity set extern_uid to the LDAP DN + # For an existing identity with matching email but changed DN, update the DN. + # For an existing identity with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid + end - # For a new user set extern_uid to the LDAP DN - # For an existing user with matching email but changed DN, update the DN. - # For an existing user with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid + gl_user.ldap_email = auth_hash.has_email? gl_user end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index ba31599432b..36e5c2670bb 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,6 +32,10 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end + def has_email? + get_info(:email).present? + end + private def info @@ -46,8 +50,8 @@ module Gitlab def username_and_email @username_and_email ||= begin - username = get_info(:username) || get_info(:nickname) - email = get_info(:email) + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence username ||= generate_username(email) if email email ||= generate_temporarily_email(username) if username diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e3d2cc65a8f..d87a72f7ba3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -111,7 +111,7 @@ module Gitlab def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users - else + else Gitlab.config.omniauth.block_auto_created_users end end @@ -135,16 +135,16 @@ module Gitlab def user_attributes # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? - username = ldap_person.username - email = ldap_person.email.first - else - username = auth_hash.username - email = auth_hash.email + username = ldap_person.username.presence + email = ldap_person.email.first.presence end + username ||= auth_hash.username + email ||= auth_hash.email + name = auth_hash.name name = ::Namespace.clean_path(username) if name.strip.empty? - + { name: name, username: ::Namespace.clean_path(username), diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f07c8b508..1633891c8a0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -38,6 +38,7 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" @@ -233,10 +234,12 @@ start_gitlab() { if [ "$gitlab_workhorse_status" = "0" ]; then echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-workhorse does this itself + # No need to remove a socket, gitlab-workhorse does this itself. + # Because gitlab-workhorse has multiple executables we need to fix + # the PATH. $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ - $app_root/../gitlab-workhorse/gitlab-workhorse \ - $gitlab_workhorse_options \ + /usr/bin/env PATH=$gitlab_workhorse_dir:$PATH \ + gitlab-workhorse $gitlab_workhorse_options \ >> $gitlab_workhorse_log 2>&1 & fi diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1937ca582b0..4e6e56ac2db 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,6 +30,9 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# The directory where the gitlab-workhorse binaries are. Usually +# /home/git/gitlab-workhorse . +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write |