From 14023c4e5a3241d06a8a021bab1183e4a9742b28 Mon Sep 17 00:00:00 2001 From: Saito Date: Fri, 30 Dec 2011 21:41:39 +0800 Subject: [PATCH 1/4] remove charencode. --- app/helpers/application_helper.rb | 1 - app/models/commit.rb | 7 +++---- app/views/commits/_text_file.html.haml | 2 +- app/views/refs/_tree.html.haml | 4 ++-- lib/graph_commit.rb | 5 ++--- lib/utils.rb | 16 +--------------- 6 files changed, 9 insertions(+), 26 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 4b00c9ab..99ef30ae 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,6 +1,5 @@ require 'digest/md5' module ApplicationHelper - include Utils::CharEncode def gravatar_icon(user_email, size = 40) gravatar_host = request.ssl? ? "https://secure.gravatar.com" : "http://www.gravatar.com" diff --git a/app/models/commit.rb b/app/models/commit.rb index c97aec28..33d3b202 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -1,5 +1,4 @@ class Commit - include Utils::CharEncode attr_accessor :commit attr_accessor :head @@ -22,7 +21,7 @@ class Commit end def safe_message - encode(message) + message end def created_at @@ -30,11 +29,11 @@ class Commit end def author_email - encode(author.email) + author.email end def author_name - encode(author.name) + author.name end def prev_commit diff --git a/app/views/commits/_text_file.html.haml b/app/views/commits/_text_file.html.haml index 22fe5582..954722b1 100644 --- a/app/views/commits/_text_file.html.haml +++ b/app/views/commits/_text_file.html.haml @@ -1,7 +1,7 @@ %table - line_old = 0 - line_new = 0 - - diff_str = encode(diff.diff) + - diff_str = diff.diff - lines_arr = diff_str.lines.to_a - lines_arr.each do |line| - next if line.match(/^--- \/dev\/null/) diff --git a/app/views/refs/_tree.html.haml b/app/views/refs/_tree.html.haml index 07bab212..b7b3fd2c 100644 --- a/app/views/refs/_tree.html.haml +++ b/app/views/refs/_tree.html.haml @@ -40,9 +40,9 @@ %h3= content.name .readme - if content.name =~ /\.(md|markdown)$/i - = markdown(encode content.data) + = markdown(content.data) - else - = simple_format(encode content.data) + = simple_format(content.data) :javascript $(function(){ diff --git a/lib/graph_commit.rb b/lib/graph_commit.rb index 1fcb9e78..54550d99 100644 --- a/lib/graph_commit.rb +++ b/lib/graph_commit.rb @@ -1,7 +1,6 @@ require "grit" class GraphCommit - include Utils::CharEncode attr_accessor :time, :space attr_accessor :refs @@ -97,13 +96,13 @@ class GraphCommit h[:parents] = self.parents.collect do |p| [p.id,0,0] end - h[:author] = encode(author.name) + h[:author] = author.name h[:time] = time h[:space] = space h[:refs] = refs.collect{|r|r.name}.join(" ") unless refs.nil? h[:id] = sha h[:date] = date - h[:message] = encode(message) + h[:message] = message h[:login] = author.email h end diff --git a/lib/utils.rb b/lib/utils.rb index 2de14761..677322c9 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -16,28 +16,14 @@ module Utils end end - module CharEncode - def encode(string) - return '' unless string - cd = CharDet.detect(string) - if cd.confidence > 0.6 - string.force_encoding(cd.encoding) - end - string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace) - rescue - "Invalid Encoding" - end - end - module Colorize - include CharEncode def colorize system_colorize(data, name) end def system_colorize(data, file_name) ft = handle_file_type(file_name) - Pygments.highlight(encode(data), :lexer => ft, :options => { :encoding => 'utf-8', :linenos => 'True' }) + Pygments.highlight(data, :lexer => ft, :options => { :encoding => 'utf-8', :linenos => 'True' }) end def handle_file_type(file_name, mime_type = nil) From 26deb7e0b580ca402ffd93d1991381f580447973 Mon Sep 17 00:00:00 2001 From: Saito Date: Fri, 30 Dec 2011 22:25:45 +0800 Subject: [PATCH 2/4] add charlock_holmes gem --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 3c7b1a4f..e240fa65 100644 --- a/Gemfile +++ b/Gemfile @@ -22,9 +22,9 @@ gem "acts_as_list" gem "rdiscount" gem "acts-as-taggable-on", "~> 2.1.0" gem "drapper" -gem "rchardet19", "~> 1.3.5" gem "resque" gem "httparty" +gem "charlock_holmes" group :assets do gem "sass-rails", "~> 3.1.0" diff --git a/Gemfile.lock b/Gemfile.lock index 09fecb88..86a69981 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,6 +77,7 @@ GEM xpath (~> 0.1.4) carrierwave (0.5.8) activesupport (~> 3.0) + charlock_holmes (0.6.8) childprocess (0.2.2) ffi (~> 1.0.6) coffee-rails (3.1.1) @@ -172,7 +173,6 @@ GEM rdoc (~> 3.4) thor (~> 0.14.6) rake (0.9.2.2) - rchardet19 (1.3.5) rdiscount (1.6.8) rdoc (3.11) json (~> 1.4) @@ -285,6 +285,7 @@ DEPENDENCIES awesome_print capybara carrierwave + charlock_holmes coffee-rails (~> 3.1.0) database_cleaner devise (= 1.5.0) @@ -302,7 +303,6 @@ DEPENDENCIES pygments.rb (= 0.2.3) rails (= 3.1.1) rails-footnotes (~> 3.7.5) - rchardet19 (~> 1.3.5) rdiscount resque rspec-rails From 98b5c3647bae81fcc010a6f95dc557b7ea6135f6 Mon Sep 17 00:00:00 2001 From: Saito Date: Fri, 30 Dec 2011 22:26:13 +0800 Subject: [PATCH 3/4] use monkey patch to trancoding content --- config/initializers/gitlabhq/20_grit_ext.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/config/initializers/gitlabhq/20_grit_ext.rb b/config/initializers/gitlabhq/20_grit_ext.rb index 35684453..d9f2c001 100644 --- a/config/initializers/gitlabhq/20_grit_ext.rb +++ b/config/initializers/gitlabhq/20_grit_ext.rb @@ -7,5 +7,23 @@ Grit::Blob.class_eval do include Utils::Colorize end +#monkey patch raw_object from string +Grit::GitRuby::Internal::RawObject.class_eval do + def content + transcoding(@content) + end + + private + def transcoding(content) + content ||= "" + detection = CharlockHolmes::EncodingDetector.detect(content) + if hash = detection + content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding] + end + content + end +end + + Grit::Git.git_timeout = GIT_OPTS["git_timeout"] Grit::Git.git_max_size = GIT_OPTS["git_max_size"] From 31f3d6efb3b4b3f4ae5ffd64455cc2e26106a61d Mon Sep 17 00:00:00 2001 From: Saito Date: Fri, 30 Dec 2011 22:49:58 +0800 Subject: [PATCH 4/4] fix travis dependency, add libicu support --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 62a5dac6..7a450b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +before_install: sudo apt-get install libicu-dev -y branches: only: - 'master'