From 5ee15661dbc2da70927f588e310315233aff6eea Mon Sep 17 00:00:00 2001 From: murphy Date: Mon, 10 Apr 2006 03:06:50 +0000 Subject: Large update: Scanners for HTML, RHTML and Nitro-XHTML added. CSS style changes/enhancments (mainly the new background color for inline code, affects all Ruby code.) Demos and tests adjusted. Plugin: new PluginHost::default method. Scanner: - New setup method - ability to re-use a scanner - ability to keep the tokens - minor changes to token caching and string flattening Encoder: Error if token content is neither String nor Symbol. HTML encoder: - more warnings for unclosed tokens - output now UTF-8 Ruby Scanner: - bug: symbols before => now do not include =; {:foo=>bar} is valid Ruby code - try to close all open tokens - constants now all with specific namespace (for speed, I hope) Styles: new :entity/en class. Test suite now gives hinted HTML output. --- lib/coderay/scanner.rb | 59 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 19 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 16998f2..eae4c0e 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -96,21 +96,21 @@ module CodeRay raise "I am only the basic Scanner class. I can't scan "\ "anything. :( Use my subclasses." if self.class == Scanner - # I love this hack. It seems to silence - # all dos/unix/mac newline problems. - code = code.gsub(/\r\n?/, "\n") if code.index ?\r - super code + super code.to_s.to_unix + @tokens = options[:tokens] if @options[:stream] warn "warning in CodeRay::Scanner.new: :stream is set, "\ "but no block was given" unless block_given? raise NotStreamableError, self unless kind_of? Streamable - @tokens = TokenStream.new(&block) + @tokens ||= TokenStream.new(&block) else warn "warning in CodeRay::Scanner.new: Block given, "\ "but :stream is #{@options[:stream]}" if block_given? - @tokens = Tokens.new + @tokens ||= Tokens.new end + + setup end # More mnemonic accessor name for the input string. @@ -122,25 +122,28 @@ module CodeRay end def string= code - code = code.gsub(/\r\n?/, "\n") if code.index ?\r + code = code.to_s.to_unix super code reset_instance end # Scans the code and returns all tokens in a Tokens object. - def tokenize options = {} - options = @options.merge({}) #options - if @options[:stream] # :stream must have been set already - reset ## what is this for? - scan_tokens @tokens, options - @tokens - else - @cached_tokens ||= scan_tokens @tokens, options - end + def tokenize new_string=nil, options = {} + options = @options.merge(options) + self.string = new_string if new_string + @cached_tokens = + if @options[:stream] # :stream must have been set already + reset unless new_string + scan_tokens @tokens, options + @tokens + else + scan_tokens @tokens, options + end end - # You can also see tokenize as a read-only attribute - alias tokens tokenize + def tokens + @cached_tokens ||= tokenize + end # Traverses the tokens. def each &block @@ -160,6 +163,14 @@ module CodeRay protected + # Can be implemented by subclasses to do some initialization + # that has to be done once per instance. + # + # Use reset for initialization that has to be done once per + # scan. + def setup + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -171,7 +182,7 @@ module CodeRay end def reset_instance - @tokens.clear + @tokens.clear unless @options[:keep_tokens] @cached_tokens = nil end @@ -211,4 +222,14 @@ surrounding code: end end +class String + # I love this hack. It seems to silence all dos/unix/mac newline problems. + def to_unix + if index ?\r + gsub(/\r\n?/, "\n") + else + self + end + end +end # vim:sw=2:ts=2:noet:tw=78 -- cgit v1.2.1