From 48e144a20829faaeca9a7db8fbc6128f1f5d7297 Mon Sep 17 00:00:00 2001
From: murphy <murphy@rubychan.de>
Date: Tue, 4 Oct 2005 04:04:07 +0000
Subject: Two new encoders: debug and xml.

encoder.rb: new token handling
encoders/statistic.rb: using new handling

ruby_helper.rb: small improvements
ruby.rb:
- escapes in subtoken
- Float detection changed
- some multi-char operators are now scanned as one token
- def and module definition handling changed

bin/coderay: improved, new interface (still in progress)

plugin.rb: more expressive load error message
---
 lib/coderay/encoders/debug.rb     | 38 +++++++++++++++++++++
 lib/coderay/encoders/statistic.rb | 25 ++++++++------
 lib/coderay/encoders/xml.rb       | 70 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 123 insertions(+), 10 deletions(-)
 create mode 100644 lib/coderay/encoders/debug.rb
 create mode 100644 lib/coderay/encoders/xml.rb

(limited to 'lib/coderay/encoders')

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
new file mode 100644
index 0000000..b084733
--- /dev/null
+++ b/lib/coderay/encoders/debug.rb
@@ -0,0 +1,38 @@
+module CodeRay
+	module Encoders
+
+		# = Debug Encoder
+		class Debug < Encoder
+
+			include Streamable
+			register_for :debug
+
+			FILE_EXTENSION = 'debug'
+
+		protected
+			def text_token text, kind
+				@out <<
+					if kind == :space
+						text
+					else
+						text = text.gsub(/[)\\]/, '\\\\\0')
+						"#{kind}(#{text})"
+					end					
+			end
+
+			def block_token action, kind
+				@out << super
+			end
+
+			def open_token kind
+				"#{kind}<"
+			end
+
+			def close_token kind
+				">"
+			end
+
+		end
+
+	end
+end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index 0685c03..cd26272 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -22,17 +22,22 @@ module CodeRay module Encoders
 			super
 		end
 		
-		def token text, type
+		def text_token text, kind
+			@real_token_count += 1 unless kind == :space
+			@type_stats[kind].count += 1
+			@type_stats[kind].size += text.size
+			@type_stats['TOTAL'].size += text.size
+		end
+		
+		# TODO Hierarchy handling
+		def block_token action, kind
+			#@content_type = kind
+			@type_stats['open/close'].count += 1
+		end
+
+		def token text, kind
+			super
 			@type_stats['TOTAL'].count += 1
-			if text.is_a? String
-				@real_token_count += 1 unless type == :space
-				@type_stats[type].count += 1
-				@type_stats[type].size += text.size
-				@type_stats['TOTAL'].size += text.size
-			else
-				@content_type = type
-				@type_stats['open/close'].count += 1
-			end
 		end
 
 		STATS = <<-STATS
diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb
new file mode 100644
index 0000000..5596f46
--- /dev/null
+++ b/lib/coderay/encoders/xml.rb
@@ -0,0 +1,70 @@
+module CodeRay
+	module Encoders
+
+		# = Debug Encoder
+		class XML < Encoder
+
+			include Streamable
+			register_for :xml
+
+			FILE_EXTENSION = 'xml'
+
+			require 'rexml/document'
+
+			DEFAULT_OPTIONS = {
+				:tab_width => 8,
+				:pretty => -1,
+				:transitive => false,
+			}
+
+		protected
+			
+			def setup options
+				@out = ''
+				@doc = REXML::Document.new
+				@doc << REXML::XMLDecl.new
+				@tab_width = options[:tab_width]
+				@root = @node = @doc.add_element('coderay-tokens')
+			end
+			
+			def finish options
+				@doc.write @out, options[:pretty], options[:transitive], true
+				@out
+			end
+			
+			def text_token text, kind
+				if kind == :space
+					token = @node
+				else
+					token = @node.add_element kind.to_s
+				end
+				text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+					case
+					when space
+						token << REXML::Text.new(space, true)
+					when tab
+						token << REXML::Text.new(tab, true)
+					when nl
+						token << REXML::Text.new(nl, true)
+					else
+						token << REXML::Text.new($&)
+					end
+				end
+			end
+
+			def open_token kind
+				@node = @node.add_element kind.to_s
+			end
+
+			def close_token kind
+				if @node == @root
+					raise 'no token to close!'
+				end
+				@node = @node.parent
+			end
+
+		end
+
+	end
+end
+ 	 
-- 
cgit v1.2.1