Merge pull request #15 from notalex/OTWO-3206

Otwo 3206

Author: Peter Degen-Portnoy

bin/string_encoder

@@ -0,0 +1,13 @@
+#! /usr/bin/env ruby
+# Replaces invalid utf-8 characters with �.
+#
+# Usage:
+# $ cat some_file | string_encoder
+
+while input = gets
+  if input.to_s.valid_encoding?
+    puts input
+  else
+    puts input.encode('UTF-8', 'binary', invalid: :replace, undef: :replace)
+  end
+end

lib/scm/adapters/abstract_adapter.rb

@@ -13,6 +13,12 @@ class << self
 			end
 		end
 
+    # Returns path to the string_encoder binary.
+    # For use with inline system commands like `run`.
+    def string_encoder
+      File.expand_path('../../../../bin/string_encoder', __FILE__)
+    end
+
 	end
 end
 

lib/scm/adapters/cvs/commits.rb

@@ -63,7 +63,7 @@ def open_log_file(opts={})
 			after = opts[:after]
 			begin
         ensure_host_key
-				run "cvsnt -d #{self.url} rlog #{opt_branch} #{opt_time(after)} '#{self.module_name}' > #{rlog_filename}"
+				run "cvsnt -d #{self.url} rlog #{opt_branch} #{opt_time(after)} '#{self.module_name}' | #{ string_encoder } > #{rlog_filename}"
 				File.open(rlog_filename, 'r') do |file|
 					yield file
 				end

lib/scm/adapters/cvs/misc.rb

@@ -67,7 +67,7 @@ def ls(path=nil)
 
 		def log(most_recent_token=nil)
       ensure_host_key
-			run "cvsnt -d #{self.url} rlog #{opt_branch} #{opt_time(most_recent_token)} '#{self.module_name}'"
+			run "cvsnt -d #{self.url} rlog #{opt_branch} #{opt_time(most_recent_token)} '#{self.module_name}' | #{ string_encoder }"
 		end
 
 		def checkout(r, local_directory)

lib/scm/adapters/git/commits.rb

@@ -80,7 +80,7 @@ def log(opts={})
 				if opts[:after] && opts[:after]==self.head_token
 					'' # Nothing new.
 				else
-					run "#{rev_list_command(opts)} | xargs -n 1 #{Scm::Parsers::GitStyledParser.whatchanged}"
+					run "#{rev_list_command(opts)} | xargs -n 1 #{Scm::Parsers::GitStyledParser.whatchanged} | #{ string_encoder }"
 				end
 			else
 				''
@@ -96,7 +96,7 @@ def open_log_file(opts={})
 					'' # Nothing new.
 				else
           begin
-					  run "#{rev_list_command(opts)} | xargs -n 1 #{Scm::Parsers::GitStyledParser.whatchanged} > #{log_filename}"
+					  run "#{rev_list_command(opts)} | xargs -n 1 #{Scm::Parsers::GitStyledParser.whatchanged} | #{ string_encoder } > #{log_filename}"
             File.open(log_filename, 'r') { |io| yield io } 
           ensure
             File.delete(log_filename) if FileTest.exist?(log_filename)

lib/scm/adapters/git/misc.rb

@@ -48,7 +48,7 @@ def checkout
 
 		# Returns an array of all branch names
 		def branches
-			run("cd '#{self.url}' && git branch").split.collect { |b| b =~ /\b(.+)$/ ; $1 }.compact
+			run("cd '#{self.url}' && git branch | #{ string_encoder }").split.collect { |b| b =~ /\b(.+)$/ ; $1 }.compact
 		end
 
 		def has_branch?(name=self.branch_name)

lib/scm/adapters/hg/commits.rb

@@ -44,7 +44,7 @@ def commits(opts={})
 
 		# Returns a single commit, including its diffs
 		def verbose_commit(token)
-			log = run("cd '#{self.url}' && hg log -v -r #{token} --style #{Scm::Parsers::HgStyledParser.verbose_style_path}")
+			log = run("cd '#{self.url}' && hg log -v -r #{token} --style #{Scm::Parsers::HgStyledParser.verbose_style_path} | #{ string_encoder }")
 			Scm::Parsers::HgStyledParser.parse(log).first
 		end
 
@@ -64,7 +64,7 @@ def each_commit(opts={})
 		# Not used by Ohloh proper, but handy for debugging and testing
 		def log(opts={})
 			after = opts[:after] || 0
-			run "cd '#{url}' && hg log -f #{trunk_only(opts)} -v -r tip:#{after}"
+			run "cd '#{url}' && hg log -f #{trunk_only(opts)} -v -r tip:#{after} | #{ string_encoder }"
 		end
 
 		# Returns a file handle to the log.
@@ -78,7 +78,7 @@ def open_log_file(opts={})
 					# As a time optimization, just create an empty file rather than fetch a log we know will be empty.
 					File.open(log_filename, 'w') { }
 				else
-					run "cd '#{url}' && hg log --verbose #{trunk_only(opts)} -r #{after || 0}:tip --style #{Scm::Parsers::HgStyledParser.verbose_style_path} > #{log_filename}"
+					run "cd '#{url}' && hg log --verbose #{trunk_only(opts)} -r #{after || 0}:tip --style #{Scm::Parsers::HgStyledParser.verbose_style_path} | #{ string_encoder } > #{log_filename}"
 				end
 				File.open(log_filename, 'r') { |io| yield io }
 			ensure

lib/scm/adapters/hg/misc.rb

@@ -10,7 +10,7 @@ def exist?
 		end
 
 		def ls_tree(token)
-			run("cd '#{path}' && hg manifest -r #{token}").split("\n")
+			run("cd '#{path}' && hg manifest -r #{token} | #{ string_encoder }").split("\n")
 		end
 
 		def export(dest_dir, token='tip')

test/data/invalid-utf-word

@@ -0,0 +1 @@
+������

test/data/sample-content

@@ -0,0 +1,32 @@
+RCS file: /cvsroot/cncms/phparticle212/admin/template.php,v
+head: 1.4
+branch:
+locks: strict
+access list:
+symbolic names:
+	pa212: 1.1.1.1
+	niuboy: 1.1.1
+keyword substitution: kv
+total revisions: 5;	selected revisions: 4
+
+description:
+----------------------------
+revision 1.4
+date: 2006/12/08 12:08:10;  author: semirock;  state: Exp;  lines: +514 -478
+����rss.php�ļ�Ĭ����ʾ20ƪ����
+
+----------------------------
+revision 1.3
+date: 2006/11/22 08:35:59;  author: niuboy;  state: Exp;  lines: +6 -2
+no message
+----------------------------
+revision 1.2
+date: 2006/11/19 19:51:28;  author: niuboy;  state: Exp;  lines: +4 -4
+no message
+
+----------------------------
+revision 1.1
+date: 2006/11/11 07:21:46;  author: niuboy;  state: Exp;
+branches:  1.1.1;
+Initial revision
+=============================================================================