From 39eb4c9f60e296f9f6c51784ef7e0a234966755d Mon Sep 17 00:00:00 2001 From: murphy Date: Tue, 1 Jun 2010 21:38:46 +0000 Subject: Added ability to scan an Array of Strings as one with Scanner#tokenize. See issue #222. --- lib/coderay/scanner.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 7d154da..64b3c83 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -147,14 +147,27 @@ module CodeRay end # Scans the code and returns all tokens in a Tokens object. - def tokenize new_string=nil, options = {} + def tokenize source = nil, options = {} options = @options.merge(options) @tokens = options[:tokens] || @tokens || Tokens.new @tokens.scanner = self if @tokens.respond_to? :scanner= - self.string = new_string if new_string - reset unless new_string + case source + when String + self.string = source + when Array + self.string = source.join + when nil + reset + else + raise ArgumentError, 'expected String, Array, or nil' + end scan_tokens @tokens, options @cached_tokens = @tokens + if source.is_a? Array + @tokens.split_into_parts(*source.map { |part| part.size }) + else + @tokens + end end # Caches the result of tokenize. -- cgit v1.2.1