From 7fc49ec13fa0719397e00b89f2c8b3f4c6f908e2 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 29 May 2017 20:28:56 +0200 Subject: Rename Gitlab::Git::EncodingHelper to Gitlab::EncodingHelper --- app/models/merge_request_diff.rb | 2 +- app/validators/dynamic_path_validator.rb | 2 +- lib/constraints/group_url_constrainer.rb | 1 - lib/constraints/project_url_constrainer.rb | 5 +- lib/constraints/user_url_constrainer.rb | 2 +- lib/gitlab/encoding_helper.rb | 62 +++++++++++++++ lib/gitlab/git/blame.rb | 2 +- lib/gitlab/git/blob.rb | 2 +- lib/gitlab/git/commit.rb | 2 +- lib/gitlab/git/diff.rb | 2 +- lib/gitlab/git/encoding_helper.rb | 64 ---------------- lib/gitlab/git/ref.rb | 2 +- lib/gitlab/git/tree.rb | 2 +- spec/lib/constraints/group_url_constrainer_spec.rb | 6 -- .../constraints/project_url_constrainer_spec.rb | 6 -- spec/lib/constraints/user_url_constrainer_spec.rb | 6 -- spec/lib/gitlab/encoding_helper_spec.rb | 88 ++++++++++++++++++++++ spec/lib/gitlab/git/encoding_helper_spec.rb | 88 ---------------------- spec/lib/gitlab/git/repository_spec.rb | 2 +- 19 files changed, 161 insertions(+), 185 deletions(-) create mode 100644 lib/gitlab/encoding_helper.rb delete mode 100644 lib/gitlab/git/encoding_helper.rb create mode 100644 spec/lib/gitlab/encoding_helper_spec.rb delete mode 100644 spec/lib/gitlab/git/encoding_helper_spec.rb diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 1bd61c1d465..e267c6839b3 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -1,7 +1,7 @@ class MergeRequestDiff < ActiveRecord::Base include Sortable include Importable - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Prevent store of diff if commits amount more then 500 COMMITS_SAFE_SIZE = 100 diff --git a/app/validators/dynamic_path_validator.rb b/app/validators/dynamic_path_validator.rb index a9b76c7c960..27ac60637fd 100644 --- a/app/validators/dynamic_path_validator.rb +++ b/app/validators/dynamic_path_validator.rb @@ -6,7 +6,7 @@ # Values are checked for formatting and exclusion from a list of illegal path # names. class DynamicPathValidator < ActiveModel::EachValidator - extend Gitlab::Git::EncodingHelper + extend Gitlab::EncodingHelper class << self def valid_user_path?(path) diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 4d75f5363c1..6fc1d56d7a0 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,7 +1,6 @@ class GroupUrlConstrainer def matches?(request) full_path = request.params[:group_id] || request.params[:id] - full_path = Gitlab::Git::EncodingHelper.encode!(full_path) return false unless DynamicPathValidator.valid_group_path?(full_path) diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index dda8ff1688f..4c0aee6c48f 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -1,10 +1,7 @@ class ProjectUrlConstrainer def matches?(request) - namespace_path = Gitlab::Git::EncodingHelper.encode!(request.params[:namespace_id]) - + namespace_path = request.params[:namespace_id] project_path = request.params[:project_id] || request.params[:id] - project_path = Gitlab::Git::EncodingHelper.encode!(project_path) - full_path = [namespace_path, project_path].join('/') return false unless DynamicPathValidator.valid_project_path?(full_path) diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 60f18746221..d16ae7f3f40 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,6 +1,6 @@ class UserUrlConstrainer def matches?(request) - full_path = Gitlab::Git::EncodingHelper.encode!(request.params[:username]) + full_path = request.params[:username] return false unless DynamicPathValidator.valid_user_path?(full_path) diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..dbe28e6bb93 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index c1b31618e0d..ef283938fe9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 297531db4cc..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index deade337354..072a8b431a1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index d41256d9a84..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url diff --git a/spec/lib/constraints/group_url_constrainer_spec.rb b/spec/lib/constraints/group_url_constrainer_spec.rb index bc767f46d8e..db680489a8d 100644 --- a/spec/lib/constraints/group_url_constrainer_spec.rb +++ b/spec/lib/constraints/group_url_constrainer_spec.rb @@ -30,12 +30,6 @@ describe GroupUrlConstrainer, lib: true do it { expect(subject.matches?(request)).to be_falsey } end - context 'invalid encoding' do - let(:request) { build_request("hi \255") } - - it { expect(subject.matches?(request)).to be_falsey } - end - context 'when the request matches a redirect route' do context 'for a root group' do let!(:redirect_route) { group.redirect_routes.create!(path: 'gitlabb') } diff --git a/spec/lib/constraints/project_url_constrainer_spec.rb b/spec/lib/constraints/project_url_constrainer_spec.rb index 4b1a1e2f607..b6884e37aa3 100644 --- a/spec/lib/constraints/project_url_constrainer_spec.rb +++ b/spec/lib/constraints/project_url_constrainer_spec.rb @@ -39,12 +39,6 @@ describe ProjectUrlConstrainer, lib: true do it { expect(subject.matches?(request)).to be_falsey } end end - - context 'invalid encoding' do - let(:request) { build_request("hi \255", "bye \255") } - - it { expect(subject.matches?(request)).to be_falsey } - end end def build_request(namespace, project, method = 'GET') diff --git a/spec/lib/constraints/user_url_constrainer_spec.rb b/spec/lib/constraints/user_url_constrainer_spec.rb index 3a53834a875..ed69b830979 100644 --- a/spec/lib/constraints/user_url_constrainer_spec.rb +++ b/spec/lib/constraints/user_url_constrainer_spec.rb @@ -30,12 +30,6 @@ describe UserUrlConstrainer, lib: true do it { expect(subject.matches?(request)).to be_falsey } end end - - context 'invalid encoding' do - let(:request) { build_request("hi \255") } - - it { expect(subject.matches?(request)).to be_falsey } - end end def build_request(username, method = 'GET') diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb new file mode 100644 index 00000000000..1482ef7132d --- /dev/null +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -0,0 +1,88 @@ +require "spec_helper" + +describe Gitlab::EncodingHelper do + let(:ext_class) { Class.new { extend Gitlab::EncodingHelper } } + let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") } + + describe '#encode!' do + [ + [ + 'leaves ascii only string as is', + 'ascii only string', + 'ascii only string' + ], + [ + 'leaves valid utf8 string as is', + 'multibyte string №∑∉', + 'multibyte string №∑∉' + ], + [ + 'removes invalid bytes from ASCII-8bit encoded multibyte string. This can occur when a git diff match line truncates in the middle of a multibyte character. This occurs after the second word in this example. The test string is as short as we can get while still triggering the error condition when not looking at `detect[:confidence]`.', + "mu ns\xC3\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi ".force_encoding('ASCII-8BIT'), + "mu ns\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi " + ] + ].each do |description, test_string, xpect| + it description do + expect(ext_class.encode!(test_string)).to eq(xpect) + end + end + + it 'leaves binary string as is' do + expect(ext_class.encode!(binary_string)).to eq(binary_string) + end + end + + describe '#encode_utf8' do + [ + [ + "encodes valid utf8 encoded string to utf8", + "λ, λ, λ".encode("UTF-8"), + "λ, λ, λ".encode("UTF-8") + ], + [ + "encodes valid ASCII-8BIT encoded string to utf8", + "ascii only".encode("ASCII-8BIT"), + "ascii only".encode("UTF-8") + ], + [ + "encodes valid ISO-8859-1 encoded string to utf8", + "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("ISO-8859-1", "UTF-8"), + "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("UTF-8") + ] + ].each do |description, test_string, xpect| + it description do + r = ext_class.encode_utf8(test_string.force_encoding('UTF-8')) + expect(r).to eq(xpect) + expect(r.encoding.name).to eq('UTF-8') + end + end + + it 'returns empty string on conversion errors' do + expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError) + end + end + + describe '#clean' do + [ + [ + 'leaves ascii only string as is', + 'ascii only string', + 'ascii only string' + ], + [ + 'leaves valid utf8 string as is', + 'multibyte string №∑∉', + 'multibyte string №∑∉' + ], + [ + 'removes invalid bytes from ASCII-8bit encoded multibyte string.', + "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'), + "Lorem ipsum\n dolor sit amet, xyàyùabcdùefg" + ] + ].each do |description, test_string, xpect| + it description do + expect(ext_class.encode!(test_string)).to eq(xpect) + end + end + end +end diff --git a/spec/lib/gitlab/git/encoding_helper_spec.rb b/spec/lib/gitlab/git/encoding_helper_spec.rb deleted file mode 100644 index 48fc817d857..00000000000 --- a/spec/lib/gitlab/git/encoding_helper_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -require "spec_helper" - -describe Gitlab::Git::EncodingHelper do - let(:ext_class) { Class.new { extend Gitlab::Git::EncodingHelper } } - let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") } - - describe '#encode!' do - [ - [ - 'leaves ascii only string as is', - 'ascii only string', - 'ascii only string' - ], - [ - 'leaves valid utf8 string as is', - 'multibyte string №∑∉', - 'multibyte string №∑∉' - ], - [ - 'removes invalid bytes from ASCII-8bit encoded multibyte string. This can occur when a git diff match line truncates in the middle of a multibyte character. This occurs after the second word in this example. The test string is as short as we can get while still triggering the error condition when not looking at `detect[:confidence]`.', - "mu ns\xC3\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi ".force_encoding('ASCII-8BIT'), - "mu ns\n Lorem ipsum dolor sit amet, consectetur adipisicing ut\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p\n {: .normal_pn}\n \n-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in\n# *Lorem ipsum\xC3\xB9l\xC3\xB9l\xC3\xA0 dolor\xC3\xB9k\xC3\xB9 sit\xC3\xA8b\xC3\xA8 N\xC3\xA8 amet b\xC3\xA0d\xC3\xAC*\n+# *consectetur\xC3\xB9l\xC3\xB9l\xC3\xA0 adipisicing\xC3\xB9k\xC3\xB9 elit\xC3\xA8b\xC3\xA8 N\xC3\xA8 sed do\xC3\xA0d\xC3\xAC*{: .italic .smcaps}\n \n \xEF\x9B\xA1 eiusmod tempor incididunt, ut\xC3\xAAn\xC3\xB9 labore et dolore. Tw\xC4\x83nj\xC3\xAC magna aliqua. Ut enim ad minim veniam\n {: .normal}\n@@ -9,5 +9,5 @@ quis nostrud\xC3\xAAt\xC3\xB9 exercitiation ullamco laboris m\xC3\xB9s\xC3\xB9k\xC3\xB9abc\xC3\xB9 nisi " - ] - ].each do |description, test_string, xpect| - it description do - expect(ext_class.encode!(test_string)).to eq(xpect) - end - end - - it 'leaves binary string as is' do - expect(ext_class.encode!(binary_string)).to eq(binary_string) - end - end - - describe '#encode_utf8' do - [ - [ - "encodes valid utf8 encoded string to utf8", - "λ, λ, λ".encode("UTF-8"), - "λ, λ, λ".encode("UTF-8") - ], - [ - "encodes valid ASCII-8BIT encoded string to utf8", - "ascii only".encode("ASCII-8BIT"), - "ascii only".encode("UTF-8") - ], - [ - "encodes valid ISO-8859-1 encoded string to utf8", - "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("ISO-8859-1", "UTF-8"), - "Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann.".encode("UTF-8") - ] - ].each do |description, test_string, xpect| - it description do - r = ext_class.encode_utf8(test_string.force_encoding('UTF-8')) - expect(r).to eq(xpect) - expect(r.encoding.name).to eq('UTF-8') - end - end - - it 'returns empty string on conversion errors' do - expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError) - end - end - - describe '#clean' do - [ - [ - 'leaves ascii only string as is', - 'ascii only string', - 'ascii only string' - ], - [ - 'leaves valid utf8 string as is', - 'multibyte string №∑∉', - 'multibyte string №∑∉' - ], - [ - 'removes invalid bytes from ASCII-8bit encoded multibyte string.', - "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'), - "Lorem ipsum\n dolor sit amet, xyàyùabcdùefg" - ] - ].each do |description, test_string, xpect| - it description do - expect(ext_class.encode!(test_string)).to eq(xpect) - end - end - end -end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index cb107c6d1f9..9fb9eb00d53 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1,7 +1,7 @@ require "spec_helper" describe Gitlab::Git::Repository, seed_helper: true do - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) } -- cgit v1.2.1