fix the issue on github #157.

directly force_encoding is wrong,
must detect the file string's encoding.
then force_encoding the string to it's encoding.
last convert it to utf-8.

5 files changed, 23 insertions, 8 deletions

diff --git a/app/helpers/commits_helper.rb b/app/helpers/commits_helper.rb
index bc95f916d15..585564aa4f8 100644
--- a/app/helpers/commits_helper.rb
+++ b/app/helpers/commits_helper.rb
@@ -1,4 +1,5 @@
 module CommitsHelper
+  include Utils::CharEncode
   def diff_line(line, line_new = 0, line_old = 0)
     full_line = html_escape(line.gsub(/\n/, ''))
     color = if line[0] == "+"
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 6d724bc825d..0884e34225c 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -1,4 +1,6 @@
 class Commit
+  include Utils::CharEncode
+
   attr_accessor :commit
   attr_accessor :head
 
@@ -20,7 +22,7 @@ class Commit
   end
 
   def safe_message
-    message.force_encoding(Encoding::UTF_8)
+    encode(message)
   end
 
   def created_at
@@ -28,10 +30,10 @@ class Commit
   end
 
   def author_email
-    author.email.force_encoding(Encoding::UTF_8)
+    encode(author.email)
   end
 
   def author_name
-    author.name.force_encoding(Encoding::UTF_8)
+    encode(author.name)
   end
 end
diff --git a/app/views/commits/_text_file.html.haml b/app/views/commits/_text_file.html.haml
index bdc740b3952..c9cb99e2147 100644
--- a/app/views/commits/_text_file.html.haml
+++ b/app/views/commits/_text_file.html.haml
@@ -2,7 +2,7 @@
 - line_new = 0
 - lines_arr = diff.diff.lines.to_a
 - lines_arr.each do |line|
-  - line.force_encoding(Encoding::UTF_8)
+  - encode(line)
   - next if line.match(/^--- \/dev\/null/)
   - next if line.match(/^--- a/)
   - next if line.match(/^\+\+\+ b/)
diff --git a/lib/graph_commit.rb b/lib/graph_commit.rb
index 18b1702250c..1fcb9e78ef0 100644
--- a/lib/graph_commit.rb
+++ b/lib/graph_commit.rb
@@ -1,6 +1,7 @@
 require "grit"
 
 class GraphCommit
+  include Utils::CharEncode
   attr_accessor :time, :space
   attr_accessor :refs
 
@@ -65,7 +66,7 @@ class GraphCommit
   # @param [GraphCommit] the commit object.
   # @param [Hash<String,GraphCommit>] map of commits
   #
-  # @return [Fixnum] max space used.  
+  # @return [Fixnum] max space used.
   def self.mark_chain(mark, commit, map)
     commit.space = mark  if commit.space == 0
     m1 = mark - 1
@@ -96,13 +97,13 @@ class GraphCommit
     h[:parents] = self.parents.collect do |p|
       [p.id,0,0]
     end
-    h[:author]  = author.name.force_encoding("UTF-8")
+    h[:author]  = encode(author.name)
     h[:time]    = time
     h[:space]   = space
     h[:refs]    = refs.collect{|r|r.name}.join(" ") unless refs.nil?
     h[:id]      = sha
     h[:date]    = date
-    h[:message] = message.force_encoding("UTF-8")
+    h[:message] = encode(message)
     h[:login]   = author.email
     h
   end
diff --git a/lib/utils.rb b/lib/utils.rb
index f43e2edd207..23f05f6f919 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -16,9 +16,20 @@ module Utils
     end
   end
 
+  module CharEncode
+    def encode(string)
+      cd = CharDet.detect(string)
+      if cd.confidence > 0.6
+        string.force_encoding(cd.encoding)
+      end
+      string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
+    end
+  end
+
   module Colorize
+    include CharEncode
     def colorize
-      system_colorize(data, name)
+      system_colorize(encode(data), name)
     end
 
     def system_colorize(data, file_name)