Merge branch 'grit_ext'

10 files changed, 31 insertions, 29 deletions

diff --git a/.travis.yml b/.travis.yml
index 62a5dac6dfe..7a450b7d005 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,4 @@
+before_install: sudo apt-get install libicu-dev -y
 branches:
   only:
     - 'master'
diff --git a/Gemfile b/Gemfile
index 3c7b1a4f91f..e240fa65b91 100644
--- a/Gemfile
+++ b/Gemfile
@@ -22,9 +22,9 @@ gem "acts_as_list"
 gem "rdiscount"
 gem "acts-as-taggable-on", "~> 2.1.0"
 gem "drapper"
-gem "rchardet19", "~> 1.3.5"
 gem "resque"
 gem "httparty"
+gem "charlock_holmes"
 
 group :assets do
   gem "sass-rails",   "~> 3.1.0"
diff --git a/Gemfile.lock b/Gemfile.lock
index 09fecb8878e..86a69981004 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -77,6 +77,7 @@ GEM
       xpath (~> 0.1.4)
     carrierwave (0.5.8)
       activesupport (~> 3.0)
+    charlock_holmes (0.6.8)
     childprocess (0.2.2)
       ffi (~> 1.0.6)
     coffee-rails (3.1.1)
@@ -172,7 +173,6 @@ GEM
       rdoc (~> 3.4)
       thor (~> 0.14.6)
     rake (0.9.2.2)
-    rchardet19 (1.3.5)
     rdiscount (1.6.8)
     rdoc (3.11)
       json (~> 1.4)
@@ -285,6 +285,7 @@ DEPENDENCIES
   awesome_print
   capybara
   carrierwave
+  charlock_holmes
   coffee-rails (~> 3.1.0)
   database_cleaner
   devise (= 1.5.0)
@@ -302,7 +303,6 @@ DEPENDENCIES
   pygments.rb (= 0.2.3)
   rails (= 3.1.1)
   rails-footnotes (~> 3.7.5)
-  rchardet19 (~> 1.3.5)
   rdiscount
   resque
   rspec-rails
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 4b00c9ab7e2..99ef30aef51 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -1,6 +1,5 @@
 require 'digest/md5'
 module ApplicationHelper
-  include Utils::CharEncode
 
   def gravatar_icon(user_email, size = 40)
     gravatar_host = request.ssl? ? "https://secure.gravatar.com" :  "http://www.gravatar.com"
diff --git a/app/models/commit.rb b/app/models/commit.rb
index c97aec28c72..33d3b202ee7 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -1,5 +1,4 @@
 class Commit
-  include Utils::CharEncode
 
   attr_accessor :commit
   attr_accessor :head
@@ -22,7 +21,7 @@ class Commit
   end
 
   def safe_message
-    encode(message)
+    message
   end
 
   def created_at
@@ -30,11 +29,11 @@ class Commit
   end
 
   def author_email
-    encode(author.email)
+    author.email
   end
 
   def author_name
-    encode(author.name)
+    author.name
   end
 
   def prev_commit
diff --git a/app/views/commits/_text_file.html.haml b/app/views/commits/_text_file.html.haml
index 22fe5582dfd..954722b1f31 100644
--- a/app/views/commits/_text_file.html.haml
+++ b/app/views/commits/_text_file.html.haml
@@ -1,7 +1,7 @@
 %table
   - line_old = 0
   - line_new = 0
-  - diff_str = encode(diff.diff)
+  - diff_str = diff.diff
   - lines_arr = diff_str.lines.to_a
   - lines_arr.each do |line|
     - next if line.match(/^--- \/dev\/null/)
diff --git a/app/views/refs/_tree.html.haml b/app/views/refs/_tree.html.haml
index 07bab2129dc..b7b3fd2ca80 100644
--- a/app/views/refs/_tree.html.haml
+++ b/app/views/refs/_tree.html.haml
@@ -40,9 +40,9 @@
         %h3= content.name
         .readme
           - if content.name =~ /\.(md|markdown)$/i
-            = markdown(encode content.data)
+            = markdown(content.data)
           - else
-            = simple_format(encode content.data)
+            = simple_format(content.data)
 
 :javascript
   $(function(){
diff --git a/config/initializers/gitlabhq/20_grit_ext.rb b/config/initializers/gitlabhq/20_grit_ext.rb
index 356844537aa..d9f2c001804 100644
--- a/config/initializers/gitlabhq/20_grit_ext.rb
+++ b/config/initializers/gitlabhq/20_grit_ext.rb
@@ -7,5 +7,23 @@ Grit::Blob.class_eval do
   include Utils::Colorize
 end
 
+#monkey patch raw_object from string
+Grit::GitRuby::Internal::RawObject.class_eval do
+  def content
+    transcoding(@content)
+  end
+
+  private
+  def transcoding(content)
+    content ||= ""
+    detection = CharlockHolmes::EncodingDetector.detect(content)
+    if hash = detection
+     content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding]
+    end
+    content
+  end
+end
+
+
 Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
 Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
diff --git a/lib/graph_commit.rb b/lib/graph_commit.rb
index 1fcb9e78ef0..54550d99cf6 100644
--- a/lib/graph_commit.rb
+++ b/lib/graph_commit.rb
@@ -1,7 +1,6 @@
 require "grit"
 
 class GraphCommit
-  include Utils::CharEncode
   attr_accessor :time, :space
   attr_accessor :refs
 
@@ -97,13 +96,13 @@ class GraphCommit
     h[:parents] = self.parents.collect do |p|
       [p.id,0,0]
     end
-    h[:author]  = encode(author.name)
+    h[:author]  = author.name
     h[:time]    = time
     h[:space]   = space
     h[:refs]    = refs.collect{|r|r.name}.join(" ") unless refs.nil?
     h[:id]      = sha
     h[:date]    = date
-    h[:message] = encode(message)
+    h[:message] = message
     h[:login]   = author.email
     h
   end
diff --git a/lib/utils.rb b/lib/utils.rb
index 2de14761444..677322c98d4 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -16,28 +16,14 @@ module Utils
     end
   end
 
-  module CharEncode
-    def encode(string)
-      return '' unless string
-      cd = CharDet.detect(string)
-      if cd.confidence > 0.6
-        string.force_encoding(cd.encoding)
-      end
-      string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
-    rescue
-      "Invalid Encoding"
-    end
-  end
-
   module Colorize
-    include CharEncode
     def colorize
       system_colorize(data, name)
     end
 
     def system_colorize(data, file_name)
       ft = handle_file_type(file_name)
-      Pygments.highlight(encode(data), :lexer => ft, :options => { :encoding => 'utf-8', :linenos => 'True' })
+      Pygments.highlight(data, :lexer => ft, :options => { :encoding => 'utf-8', :linenos => 'True' })
     end
 
     def handle_file_type(file_name, mime_type = nil)