From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001 From: Quintus Date: Sun, 22 Apr 2012 22:10:42 +0200 Subject: Lua scanner for CodeRay. Meta-commit. This commit is a super-commit containing all the subcommits for implementing the Lua scanner. --- lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++++++++++ lib/coderay/styles/alpha.rb | 3 + lib/coderay/token_kinds.rb | 1 + 3 files changed, 271 insertions(+) create mode 100644 lib/coderay/scanners/lua.rb (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 0000000..e7706fc --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +# Scanner for the Lua[http://lua.org] programming lanuage. +# +# The language’s complete syntax is defined in +# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], +# which is what this scanner tries to conform to. +class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident) + .add(KEYWORDS, :keyword) + .add(PREDEFINED_CONSTANTS, :predefined_constant) + .add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + @encoder = encoder + @options = options + + send(:"handle_state_#@state") until eos? + + @encoder + end + + def handle_state_initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:comment) + @encoder.text_token(match, :delimiter) + @state = :long_comment + + elsif match = scan(/--.*?$/) # --Lua comment + @encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + @encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + @encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + @state = :function_expected + elsif kind == :keyword and match == "goto" + @state = :goto_label_expected + elsif kind == :keyword and match == "local" + @state = :local_var_expected + end + + @encoder.text_token(match, kind) + + elsif match = scan(/{/) # Opening table brace { + @encoder.begin_group(:table) + @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + @state = :table + + elsif match = scan(/}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + @encoder.text_token(match, :delimiter) + elsif @brace_depth == 0 # Mismatched brace + @encoder.text_token(match, :error) + else + @brace_depth -= 1 + @encoder.text_token(match, :inline_delimiter) + @state = :table + end + @encoder.end_group(:table) + + elsif match = scan(/["']/) # String delimiters " and ' + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @start_delim = match + @state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + @encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + @encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + @encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :table scanning state. + @state = :table if @state == :initial && @brace_depth >= 1 + end + + def handle_state_function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + @encoder.text_token(match, :operator) + @state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + @encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + @encoder.text_token(match, :function) + @state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + @state = :initial + end + end + + def handle_state_goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :label) + @state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_local_var_expected + if match = scan(/function/) # local function ... + @encoder.text_token(match, :keyword) + @state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + @encoder.text_token(match, :operator) + elsif match = scan(/=/) + @encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + @state = :initial + elsif match = scan(/\n/) + @encoder.text_token(match, :space) + @state = :initial + elsif match = scan(/\s+/) + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:comment) + @state = :initial + end + + def handle_state_long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:string) + @state = :initial + end + + def handle_state_string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + @encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + @encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + @encoder.text_token(match, :delimiter) + @encoder.end_group(:string) + @state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + @encoder.end_group(:string) + @state = :initial + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_table + if match = scan(/[,;]/) + @encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + @encoder.text_token(match, :key) + @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + @encoder.text_token(scan(/=/), :operator) + @state = :initial + elsif match = scan(/\s+/m) + @encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + @state = :initial + end + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 8506d10..257083e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -116,6 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } +.table .content { color:#808 } +.table .delimiter { color:#40A} +.table { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e..e245623 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -63,6 +63,7 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', + :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', -- cgit v1.2.1 From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 17:45:49 +0200 Subject: use case+when instead of send and methods --- lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 33 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..e712061 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -57,13 +57,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - - send(:"handle_state_#@state") until eos? - - @encoder - end - - def handle_state_initial + + until eos? + case state + + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @encoder.begin_group(:comment) @@ -99,13 +97,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) @@ -146,9 +144,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - end - - def handle_state_function_expected + + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @state = :initial @@ -163,9 +160,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) @state = :initial end - end - def handle_state_goto_label_expected + when :goto_label_expected if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) @encoder.text_token(match, :label) @state = :initial @@ -174,9 +170,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_local_var_expected + + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @state = :function_expected @@ -198,9 +193,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_long_comment + + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -212,9 +206,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - end - - def handle_state_long_string + + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -226,9 +219,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - end - - def handle_state_string + + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) @@ -244,9 +236,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_table + + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @@ -262,6 +253,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # advances the pointer). @state = :initial end + else + raise + end + + end + + @encoder end end -- cgit v1.2.1 From 2b16d115f7d6caf21864934df763556e126d6357 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 18:05:04 +0200 Subject: fix for Ruby 1.8, escape { and } in regexps --- lib/coderay/scanners/lua.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..8464ef7 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -99,13 +99,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) -- cgit v1.2.1 From b9cf7f67d869eead647e237d963ba2db2fdfbc4f Mon Sep 17 00:00:00 2001 From: Joel Holdbrooks Date: Tue, 3 Jul 2012 18:17:56 -0700 Subject: add "id" token kind --- lib/coderay/token_kinds.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/coderay') diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e..41a89e4 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -39,6 +39,7 @@ module CodeRay :function => 'function', :global_variable => 'global-variable', :hex => 'hex', + :id => 'id', :imaginary => 'imaginary', :important => 'important', :include => 'include', -- cgit v1.2.1 From e796a247e5bf2395ec72b0ae98d68bcb648f5368 Mon Sep 17 00:00:00 2001 From: Joel Holdbrooks Date: Tue, 3 Jul 2012 18:19:05 -0700 Subject: use "id" instead of "constant" for css ids --- lib/coderay/scanners/css.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 7b731ef..c473574 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -8,7 +8,7 @@ module Scanners KINDS_NOT_LOC = [ :comment, :class, :pseudo_class, :type, - :constant, :directive, + :id, :directive, :key, :value, :operator, :color, :float, :string, :error, :important, ] # :nodoc: @@ -73,7 +73,7 @@ module Scanners encoder.text_token match, :class next elsif match = scan(RE::Id) - encoder.text_token match, :constant + encoder.text_token match, :id next elsif match = scan(RE::PseudoClass) encoder.text_token match, :pseudo_class -- cgit v1.2.1 From abb92f30b12e11781afa76f43a344627520b5b34 Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Sun, 8 Jul 2012 14:32:28 +0800 Subject: New: *Go Encoder* Draft version, copy from c --- lib/coderay/helpers/file_type.rb | 37 ++++---- lib/coderay/scanners/go.rb | 195 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 lib/coderay/scanners/go.rb (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 637001b..5159054 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -86,6 +86,7 @@ module CodeRay 'dpr' => :delphi, 'erb' => :erb, 'gemspec' => :ruby, + 'go' => :go, 'groovy' => :groovy, 'gvy' => :groovy, 'h' => :c, @@ -128,16 +129,16 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb new file mode 100644 index 0000000..4431ef2 --- /dev/null +++ b/lib/coderay/scanners/go.rb @@ -0,0 +1,195 @@ +module CodeRay +module Scanners + + # Scanner for Go, copy from c + class Go < Scanner + + register_for :go + file_extension 'go' + + # http://golang.org/ref/spec#Keywords + KEYWORDS = [ + 'break', 'default', 'func', 'interface', 'select', + 'case', 'defer', 'go', 'map', 'struct', + 'chan', 'else', 'goto', 'package', 'switch', + 'const', 'fallthrough', 'if', 'range', 'type', + 'continue', 'for', 'import', 'return', 'var', + ] # :nodoc: + + # http://golang.org/ref/spec#Types + PREDEFINED_TYPES = [ + 'bool', + 'uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', + 'float32', 'float64', + 'complex64', 'complex128', + 'byte', 'rune', + 'uint', 'int', 'uintptr', + ] # :nodoc: + + PREDEFINED_CONSTANTS = [ + 'nil', 'iota', + 'true', 'false', + ] # :nodoc: + + DIRECTIVES = [ + 'go_no_directive', # Seems no directive concept in Go? + ] # :nodoc: + + IDENT_KIND = WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_TYPES, :predefined_type). + add(DIRECTIVES, :directive). + add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + + protected + + def scan_tokens encoder, options + + state = :initial + label_expected = true + case_expected = false + label_expected_before_preproc_line = nil + in_preproc_line = false + + until eos? + + case state + + when :initial + + if match = scan(/ \s+ | \\\n /x) + if in_preproc_line && match != "\\\n" && match.index(?\n) + in_preproc_line = false + label_expected = label_expected_before_preproc_line + end + encoder.text_token match, :space + + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + encoder.text_token match, :comment + + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + label_expected = match =~ /[;\{\}]/ + if case_expected + label_expected = true if match == ':' + case_expected = false + end + encoder.text_token match, :operator + + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) + kind = IDENT_KIND[match] + if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) + kind = :label + match << matched + else + label_expected = false + if kind == :keyword + case match + when 'case', 'default' + case_expected = true + end + end + end + encoder.text_token match, kind + + elsif match = scan(/L?"/) + encoder.begin_group :string + if match[0] == ?L + encoder.text_token 'L', :modifier + match = '"' + end + encoder.text_token match, :delimiter + state = :string + + elsif match = scan(/ \# \s* if \s* 0 /x) + match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? + encoder.text_token match, :comment + + elsif match = scan(/#[ \t]*(\w*)/) + encoder.text_token match, :preprocessor + in_preproc_line = true + label_expected_before_preproc_line = label_expected + state = :include_expected if self[1] == 'include' + + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + label_expected = false + encoder.text_token match, :char + + elsif match = scan(/\$/) + encoder.text_token match, :ident + + elsif match = scan(/0[xX][0-9A-Fa-f]+/) + label_expected = false + encoder.text_token match, :hex + + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + label_expected = false + encoder.text_token match, :octal + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer + + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + label_expected = false + encoder.text_token match, :float + + else + encoder.text_token getch, :error + + end + + when :string + if match = scan(/[^\\\n"]+/) + encoder.text_token match, :content + elsif match = scan(/"/) + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + label_expected = false + elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + encoder.text_token match, :char + elsif match = scan(/ \\ | $ /x) + encoder.end_group :string + encoder.text_token match, :error + state = :initial + label_expected = false + else + raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + end + + when :include_expected + if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) + encoder.text_token match, :include + state = :initial + + elsif match = scan(/\s+/) + encoder.text_token match, :space + state = :initial if match.index ?\n + + else + state = :initial + + end + + else + raise_inspect 'Unknown state', encoder + + end + + end + + if state == :string + encoder.end_group :string + end + + encoder + end + + end + +end +end -- cgit v1.2.1 From 4eb4324c6fb20b9b5e7f73f17bc21e4a17fd3073 Mon Sep 17 00:00:00 2001 From: Joel Holdbrooks Date: Tue, 24 Jul 2012 10:59:50 -0700 Subject: use :tag instead of :type for identifiers --- lib/coderay/scanners/css.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index c473574..503e14e 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -7,7 +7,7 @@ module Scanners KINDS_NOT_LOC = [ :comment, - :class, :pseudo_class, :type, + :class, :pseudo_class, :tag, :id, :directive, :key, :value, :operator, :color, :float, :string, :error, :important, @@ -67,7 +67,7 @@ module Scanners elsif case states.last when :initial, :media if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox) - encoder.text_token match, :type + encoder.text_token match, :tag next elsif match = scan(RE::Class) encoder.text_token match, :class @@ -102,7 +102,7 @@ module Scanners when :media_before_name if match = scan(RE::Ident) - encoder.text_token match, :type + encoder.text_token match, :tag states[-1] = :media_after_name next end -- cgit v1.2.1 From ec39f80865cac97a49a6fec138dedc22e5971bad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 28 Jul 2012 13:12:37 +0200 Subject: fix whitespace --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5159054..5cc7446 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -129,16 +129,16 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end -- cgit v1.2.1 From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Tue, 11 Sep 2012 15:04:45 -0600 Subject: escape = in regex (my editor gets confused:-) --- lib/coderay/scanners/lua.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e712061..e640397 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -57,10 +57,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - + until eos? case state - + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @@ -144,7 +144,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @@ -170,7 +170,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @@ -179,7 +179,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, :local_variable) elsif match = scan(/,/) @encoder.text_token(match, :operator) - elsif match = scan(/=/) + elsif match = scan(/\=/) @encoder.text_token(match, :operator) # After encountering the equal sign, arbitrary expressions are # allowed again, so just return to the main state for further @@ -193,7 +193,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -206,7 +206,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -219,7 +219,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) @@ -236,14 +236,14 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @encoder.text_token(match, :key) @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/=/), :operator) + @encoder.text_token(scan(/\=/), :operator) @state = :initial elsif match = scan(/\s+/m) @encoder.text_token(match, :space) -- cgit v1.2.1 From aeac6672fbe70702288b259605cb863493ad8ac2 Mon Sep 17 00:00:00 2001 From: SHIMOMURA Sho Date: Sun, 21 Oct 2012 01:23:56 +0900 Subject: TaskPaper scanner for CodeRay --- lib/coderay/scanners/taskpaper.rb | 34 ++++++++++++++++++++++++++++++++++ lib/coderay/styles/alpha.rb | 3 +++ lib/coderay/token_kinds.rb | 3 +++ 3 files changed, 40 insertions(+) create mode 100644 lib/coderay/scanners/taskpaper.rb (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb new file mode 100644 index 0000000..c6eb3b3 --- /dev/null +++ b/lib/coderay/scanners/taskpaper.rb @@ -0,0 +1,34 @@ +module CodeRay +module Scanners + + class Taskpaper < Scanner + + register_for :taskpaper + file_extension 'taskpaper' + + protected + + def scan_tokens encoder, options + until eos? + if match = scan(/^\S.*:.*$/) # project + encoder.text_token(match, :project) + elsif match = scan(/^-.+@done(?:\(.*)?.*$/) # completed task + encoder.text_token(match, :complete) + elsif match = scan(/^-.+$/) # task + encoder.text_token(match, :plain) + elsif match = scan(/^.+$/) # comment + encoder.text_token(match, :comment) + elsif match = scan(/\s+/) # space + encoder.text_token(match, :space) + else # other + encoder.text_token getch, :error + end + end + + encoder + end + + end + +end +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 8506d10..259e458 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -134,6 +134,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .delete .delete { color: #c00; background:transparent; font-weight:bold } .change .change { color: #88f } .head .head { color: #f4f } + +.project { color: #707; font-weight: bold } +.complete { text-decoration: line-through; color: gray } TOKENS end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e..a36d877 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -74,6 +74,9 @@ module CodeRay :insert => 'insert', :eyecatcher => 'eyecatcher', + + :project => 'project', + :complete => 'complete', :ident => false, :operator => false, -- cgit v1.2.1 From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sat, 27 Oct 2012 23:41:52 -0600 Subject: :map token kind Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style. --- lib/coderay/scanners/lua.rb | 14 +++++++------- lib/coderay/styles/alpha.rb | 8 ++++---- lib/coderay/token_kinds.rb | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e640397..64763dc 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -98,10 +98,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:table) + @encoder.begin_group(:map) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 - @state = :table + @state = :map elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @@ -112,9 +112,9 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @brace_depth -= 1 @encoder.text_token(match, :inline_delimiter) - @state = :table + @state = :map end - @encoder.end_group(:table) + @encoder.end_group(:map) elsif match = scan(/["']/) # String delimiters " and ' @encoder.begin_group(:string) @@ -142,8 +142,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). - # If this is the case, return to :table scanning state. - @state = :table if @state == :initial && @brace_depth >= 1 + # If this is the case, return to :map scanning state. + @state = :map if @state == :initial && @brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -237,7 +237,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) end - when :table + when :map if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 257083e..a60725b 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style @@ -116,9 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } -.table .content { color:#808 } -.table .delimiter { color:#40A} -.table { background-color:hsla(200,100%,50%,0.06); } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index e245623..a0ad5e4 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :annotation => 'annotation', :attribute_name => 'attribute-name', @@ -50,6 +50,7 @@ module CodeRay :keyword => 'keyword', :label => 'label', :local_variable => 'local-variable', + :map => 'map', :modifier => 'modifier', :namespace => 'namespace', :octal => 'octal', @@ -63,29 +64,28 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', - :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', :variable => 'variable', - + :change => 'change', :delete => 'delete', :head => 'head', :insert => 'insert', - + :eyecatcher => 'eyecatcher', - + :ident => false, :operator => false, - + :space => false, :plain => false ) - + TokenKinds[:method] = TokenKinds[:function] TokenKinds[:escape] = TokenKinds[:delimiter] TokenKinds[:docstring] = TokenKinds[:comment] - + TokenKinds.freeze end -- cgit v1.2.1 From 51526173dd7ee640171ee597d3c380c5dda42762 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 10 Mar 2013 22:40:10 +0100 Subject: support multi-line comments in JavaScript diff --- lib/coderay/scanners/java_script.rb | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 43ecb18..75881a0 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -54,10 +54,17 @@ module Scanners protected + def setup + @state = :initial + end + def scan_tokens encoder, options - state = :initial - string_delimiter = nil + state, string_delimiter = options[:state] || @state + if string_delimiter + encoder.begin_group state + end + value_expected = true key_expected = false function_expected = false @@ -72,9 +79,10 @@ module Scanners value_expected = true if !value_expected && match.index(?\n) encoder.text_token match, :space - elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx) value_expected = true encoder.text_token match, :comment + state = :open_multi_line_comment if self[1] elsif check(/\.?\d/) key_expected = value_expected = false @@ -182,13 +190,28 @@ module Scanners raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end + when :open_multi_line_comment + if match = scan(%r! .*? \*/ !mx) + state = :initial + else + match = scan(%r! .+ !mx) + end + value_expected = true + encoder.text_token match, :comment if match + else - raise_inspect 'Unknown state', encoder + #:nocov: + raise_inspect 'Unknown state: %p' % [state], encoder + #:nocov: end end + if options[:keep_state] + @state = state, string_delimiter + end + if [:string, :regexp].include? state encoder.end_group state end -- cgit v1.2.1 From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:19 -0600 Subject: Support CDATA blocks in HTML/XML --- lib/coderay/scanners/html.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..fcf249a 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -99,7 +99,17 @@ module Scanners case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :content + encoder.text_token ']]>', :delimiter + else + encoder.text_token scan(/.*/m), :error + end + encoder.end_group :string + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype -- cgit v1.2.1 From 5a833b265c5cb918f906fb1047c0dc65e539260f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 06:09:59 +0200 Subject: always load the plugin map --- lib/coderay/helpers/plugin.rb | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index dd4e830..d14c5a9 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -131,7 +131,7 @@ module CodeRay # A Hash of plugion_id => Plugin pairs. def plugin_hash - @plugin_hash ||= make_plugin_hash + @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } end # Returns an array of all .rb files in the plugin path. @@ -158,7 +158,6 @@ module CodeRay # This is done automatically when plugin_path is called. def load_plugin_map mapfile = path_to '_map' - @plugin_map_loaded = true if File.exist? mapfile require mapfile true @@ -171,22 +170,16 @@ module CodeRay # Return a plugin hash that automatically loads plugins. def make_plugin_hash - @plugin_map_loaded ||= false Hash.new do |h, plugin_id| id = validate_id(plugin_id) path = path_to id begin require path rescue LoadError => boom - if @plugin_map_loaded - if h.has_key?(:default) - h[:default] - else - raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] - end + if h.has_key?(:default) + h[:default] else - load_plugin_map - h[plugin_id] + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] end else # Plugin should have registered by now @@ -270,7 +263,6 @@ module CodeRay end def aliases - plugin_host.load_plugin_map plugin_host.plugin_hash.inject [] do |aliases, (key, _)| aliases << key if plugin_host[key] == self aliases -- cgit v1.2.1 From e2acec3ef141725d2fc264e56d1aa18e838c6acf Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 15:23:52 +0200 Subject: massively improve Terminal encoder speed --- lib/coderay/encoders/terminal.rb | 141 +++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 74 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index a0ceb3c..500e5d8 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -19,73 +19,73 @@ module CodeRay register_for :terminal TOKEN_COLORS = { - :annotation => '35', - :attribute_name => '33', - :attribute_value => '31', - :binary => '1;35', + :annotation => "\e[35m", + :attribute_name => "\e[33m", + :attribute_value => "\e[31m", + :binary => "\e[1;35m", :char => { - :self => '36', :delimiter => '1;34' + :self => "\e[36m", :delimiter => "\e[1;34m" }, - :class => '1;35', - :class_variable => '36', - :color => '32', - :comment => '37', - :complex => '1;34', - :constant => ['1;34', '4'], - :decoration => '35', - :definition => '1;32', - :directive => ['32', '4'], - :doc => '46', - :doctype => '1;30', - :doc_string => ['31', '4'], - :entity => '33', - :error => ['1;33', '41'], - :exception => '1;31', - :float => '1;35', - :function => '1;34', - :global_variable => '42', - :hex => '1;36', - :include => '33', - :integer => '1;34', - :key => '35', - :label => '1;15', - :local_variable => '33', - :octal => '1;35', - :operator_name => '1;29', - :predefined_constant => '1;36', - :predefined_type => '1;30', - :predefined => ['4', '1;34'], - :preprocessor => '36', - :pseudo_class => '1;34', + :class => "\e[1;35m", + :class_variable => "\e[36m", + :color => "\e[32m", + :comment => "\e[37m", + :complex => "\e[1;34m", + :constant => "\e[1;34m\e[4m", + :decoration => "\e[35m", + :definition => "\e[1;32m", + :directive => "\e[32m\e[4m", + :doc => "\e[46m", + :doctype => "\e[1;30m", + :doc_string => "\e[31m\e[4m", + :entity => "\e[33m", + :error => "\e[1;33m\e[41m", + :exception => "\e[1;31m", + :float => "\e[1;35m", + :function => "\e[1;34m", + :global_variable => "\e[42m", + :hex => "\e[1;36m", + :include => "\e[33m", + :integer => "\e[1;34m", + :key => "\e[35m", + :label => "\e[1;15m", + :local_variable => "\e[33m", + :octal => "\e[1;35m", + :operator_name => "\e[1;29m", + :predefined_constant => "\e[1;36m", + :predefined_type => "\e[1;30m", + :predefined => "\e[4m\e[1;34m", + :preprocessor => "\e[36m", + :pseudo_class => "\e[1;34m", :regexp => { - :self => '31', - :content => '31', - :delimiter => '1;29', - :modifier => '35', + :self => "\e[31m", + :content => "\e[31m", + :delimiter => "\e[1;29m", + :modifier => "\e[35m", }, - :reserved => '1;31', + :reserved => "\e[1;31m", :shell => { - :self => '42', - :content => '1;29', - :delimiter => '37', + :self => "\e[42m", + :content => "\e[1;29m", + :delimiter => "\e[37m", }, :string => { - :self => '32', - :modifier => '1;32', - :escape => '1;36', - :delimiter => '1;32', - :char => '1;36', + :self => "\e[32m", + :modifier => "\e[1;32m", + :escape => "\e[1;36m", + :delimiter => "\e[1;32m", + :char => "\e[1;36m", }, - :symbol => '1;32', - :tag => '1;34', - :type => '1;34', - :value => '36', - :variable => '1;34', + :symbol => "\e[1;32m", + :tag => "\e[1;34m", + :type => "\e[1;34m", + :value => "\e[36m", + :variable => "\e[1;34m", - :insert => '42', - :delete => '41', - :change => '44', - :head => '45' + :insert => "\e[42m", + :delete => "\e[41m", + :change => "\e[44m", + :head => "\e[45m" } TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] @@ -114,10 +114,10 @@ module CodeRay end end - @out << ansi_colorize(color) - @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color)) - @out << ansi_clear - @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self] + @out << color + @out << text.gsub("\n", "\e[0m\n" + color) + @out << "\e[0m" + @out << @subcolors[:self] if @subcolors else @out << text end @@ -134,7 +134,7 @@ module CodeRay # nothing to close else @opened.pop - @out << ansi_clear + @out << "\e[0m" @out << open_token(@opened.last) end end @@ -146,7 +146,7 @@ module CodeRay @opened.pop # whole lines to be highlighted, # eg. added/modified/deleted lines in a diff - @out << "\t" * 100 + ansi_clear + @out << (@line_filler ||= "\t" * 100 + "\e[0m") @out << open_token(@opened.last) end end @@ -157,23 +157,16 @@ module CodeRay if color = TOKEN_COLORS[kind] if Hash === color @subcolors = color - ansi_colorize(color[:self]) if color[:self] + color[:self] else @subcolors = {} - ansi_colorize(color) + color end else @subcolors = nil '' end end - - def ansi_colorize(color) - Array(color).map { |c| "\e[#{c}m" }.join - end - def ansi_clear - ansi_colorize(0) - end end end -end \ No newline at end of file +end -- cgit v1.2.1 From 9d41587459ec6de14993e62c41666b0cc2692e9f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 16:06:08 +0200 Subject: cleanup HTML encoder (#135) --- lib/coderay/encoders/html.rb | 66 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 0fd1317..2c66aab 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -126,22 +126,21 @@ module Encoders protected - HTML_ESCAPE = { #:nodoc: - '&' => '&', - '"' => '"', - '>' => '>', - '<' => '<', - } + def self.make_html_escape_hash + { + '&' => '&', + '"' => '"', + '>' => '>', + '<' => '<', + # "\t" => will be set to ' ' * options[:tab_width] during setup + }.tap do |hash| + # Escape ASCII control codes except \x9 == \t and \xA == \n. + (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' } + end + end - # This was to prevent illegal HTML. - # Strange chars should still be avoided in codes. - evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s] - evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' } - #ansi_chars = Array(0x7f..0xff) - #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i } - # \x9 (\t) and \xA (\n) not included - #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/ - HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/ + HTML_ESCAPE = make_html_escape_hash + HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } @@ -255,20 +254,10 @@ module Encoders public def text_token text, kind - if text =~ /#{HTML_ESCAPE_PATTERN}/o - text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } - end - style = @span_for_kind[@last_opened ? [kind, *@opened] : kind] - if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0 - close = '' * c - reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '') - end - text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}") - end + text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o + text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n") if style @out << style << text << '' @@ -285,9 +274,7 @@ module Encoders end def end_group kind - if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind) - warn 'Malformed token stream: Trying to close a token group (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]] - end + check_group_nesting 'token group', kind if $CODERAY_DEBUG if @opened.pop @out << '' @last_opened = @opened.last if @last_opened @@ -310,15 +297,28 @@ module Encoders end def end_line kind - if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind) - warn 'Malformed token stream: Trying to close a line (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]] - end + check_group_nesting 'line', kind if $CODERAY_DEBUG if @opened.pop @out << '' @last_opened = @opened.last if @last_opened end end + protected + + def check_group_nesting name, kind + if @opened.empty? || @opened.last != kind + warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]] + end + end + + def break_lines text, style + reopen = '' + @opened.each_with_index do |k, index| + reopen << (@span_for_kind[index > 0 ? [k, *@opened[0...index]] : k] || '') + end + text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") + end end end -- cgit v1.2.1 From c39168e8a8f8313a4e3b6a9f070f14b2cf5e18a7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 17:17:07 +0200 Subject: cleanup HTML encoder #setup (#135) --- lib/coderay/encoders/html.rb | 70 ++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 38 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 2c66aab..f0f5d62 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -171,13 +171,14 @@ module Encoders def setup options super + check_options options + if options[:wrap] || options[:line_numbers] @real_out = @out @out = '' end options[:break_lines] = true if options[:line_numbers] == :inline - @break_lines = (options[:break_lines] == true) @HTML_ESCAPE = HTML_ESCAPE.dup @@ -187,43 +188,7 @@ module Encoders @last_opened = nil @css = CSS.new options[:style] - hint = options[:hint] - if hint && ![:debug, :info, :info_long].include?(hint) - raise ArgumentError, "Unknown value %p for :hint; \ - expected :info, :info_long, :debug, false, or nil." % hint - end - - css_classes = TokenKinds - case options[:css] - when :class - @span_for_kind = Hash.new do |h, k| - if k.is_a? ::Symbol - kind = k_dup = k - else - kind = k.first - k_dup = k.dup - end - if kind != :space && (hint || css_class = css_classes[kind]) - title = HTML.token_path_to_hint hint, k if hint - css_class ||= css_classes[kind] - h[k_dup] = "" - else - h[k_dup] = nil - end - end - when :style - @span_for_kind = Hash.new do |h, k| - kind = k.is_a?(Symbol) ? k : k.first - h[k.is_a?(Symbol) ? k : k.dup] = - if kind != :space && (hint || css_classes[kind]) - title = HTML.token_path_to_hint hint, k if hint - style = @css.get_style Array(k).map { |c| css_classes[c] } - "" - end - end - else - raise ArgumentError, "Unknown value %p for :css." % options[:css] - end + @span_for_kind = make_span_for_kind(options[:css], options[:hint], @css) @set_last_opened = options[:hint] || options[:css] == :style end @@ -306,6 +271,35 @@ module Encoders protected + def check_options options + unless [false, nil, :debug, :info, :info_long].include? options[:hint] + raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]] + end + + unless [:class, :style].include? options[:css] + raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]] + end + end + + def make_span_for_kind method, hint, css + css_classes = TokenKinds + + Hash.new do |h, k| + kind = k.is_a?(Symbol) ? k : k.first + + h[k.is_a?(Symbol) ? k : k.dup] = + if kind != :space && ((css_class = css_classes[kind]) || hint) + title = HTML.token_path_to_hint hint, k if hint + if method == :class + "" + else + style = css.get_style k.is_a?(Array) ? k.map { |c| css_classes[c] } : [css_class] + "" + end + end + end + end + def check_group_nesting name, kind if @opened.empty? || @opened.last != kind warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]] -- cgit v1.2.1 From a8a17fc78987bfd4236c929fd54c58f8014fe88d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 17:35:17 +0200 Subject: simplify more HTML encoder methods (#135) --- lib/coderay/encoders/html.rb | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index f0f5d62..a6c0a8e 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -171,18 +171,16 @@ module Encoders def setup options super - check_options options + check_options! options if options[:wrap] || options[:line_numbers] @real_out = @out @out = '' end - options[:break_lines] = true if options[:line_numbers] == :inline @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.dup - @HTML_ESCAPE["\t"] = ' ' * options[:tab_width] + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) @opened = [] @last_opened = nil @@ -271,7 +269,7 @@ module Encoders protected - def check_options options + def check_options! options unless [false, nil, :debug, :info, :info_long].include? options[:hint] raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]] end @@ -279,24 +277,23 @@ module Encoders unless [:class, :style].include? options[:css] raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]] end + + options[:break_lines] = true if options[:line_numbers] == :inline end def make_span_for_kind method, hint, css - css_classes = TokenKinds - - Hash.new do |h, k| - kind = k.is_a?(Symbol) ? k : k.first - - h[k.is_a?(Symbol) ? k : k.dup] = - if kind != :space && ((css_class = css_classes[kind]) || hint) - title = HTML.token_path_to_hint hint, k if hint - if method == :class - "" - else - style = css.get_style k.is_a?(Array) ? k.map { |c| css_classes[c] } : [css_class] - "" - end - end + Hash.new do |h, kinds| + h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + css_class = TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first] + title = HTML.token_path_to_hint hint, kinds if hint + + if method == :style + style = css.get_style(kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]) + "" + else + "" + end if css_class || title + end end end -- cgit v1.2.1 From cd7433c5a4d9114e8a398d4fd3eeeadecf290260 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 17:52:33 +0200 Subject: cleanup HTML encoder CSS styles/classes algorithm (#135) --- lib/coderay/encoders/html.rb | 54 ++++++++++++++++++++++++---------------- lib/coderay/encoders/html/css.rb | 14 +++++------ 2 files changed, 40 insertions(+), 28 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index a6c0a8e..b897f5e 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -186,7 +186,7 @@ module Encoders @last_opened = nil @css = CSS.new options[:style] - @span_for_kind = make_span_for_kind(options[:css], options[:hint], @css) + @span_for_kinds = make_span_for_kinds(options[:css], options[:hint]) @set_last_opened = options[:hint] || options[:css] == :style end @@ -217,7 +217,7 @@ module Encoders public def text_token text, kind - style = @span_for_kind[@last_opened ? [kind, *@opened] : kind] + style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind] text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n") @@ -231,22 +231,19 @@ module Encoders # token groups, eg. strings def begin_group kind - @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '') + @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '') @opened << kind @last_opened = kind if @set_last_opened end def end_group kind check_group_nesting 'token group', kind if $CODERAY_DEBUG - if @opened.pop - @out << '' - @last_opened = @opened.last if @last_opened - end + close_span end # whole lines to be highlighted, eg. a deleted line in a diff def begin_line kind - if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind] + if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind] if style['class="'] @out << style.sub('class="', 'class="line ') else @@ -261,10 +258,7 @@ module Encoders def end_line kind check_group_nesting 'line', kind if $CODERAY_DEBUG - if @opened.pop - @out << '' - @last_opened = @opened.last if @last_opened - end + close_span end protected @@ -281,18 +275,29 @@ module Encoders options[:break_lines] = true if options[:line_numbers] == :inline end - def make_span_for_kind method, hint, css + def css_class_for_kinds kinds + TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first] + end + + def style_for_kinds kinds + css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]] + @css.get_style_for_css_classes css_classes + end + + def make_span_for_kinds method, hint Hash.new do |h, kinds| h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin - css_class = TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first] + css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint - if method == :style - style = css.get_style(kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]) - "" - else - "" - end if css_class || title + if css_class || title + if method == :style + style = style_for_kinds(kinds) + "" + else + "" + end + end end end end @@ -306,10 +311,17 @@ module Encoders def break_lines text, style reopen = '' @opened.each_with_index do |k, index| - reopen << (@span_for_kind[index > 0 ? [k, *@opened[0...index]] : k] || '') + reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '') end text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") end + + def close_span + if @opened.pop + @out << '' + @last_opened = @opened.last if @last_opened + end + end end end diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index 6de4b46..164d7f8 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -11,7 +11,7 @@ module Encoders end def initialize style = :default - @classes = Hash.new + @styles = Hash.new style = CSS.load_stylesheet style @stylesheet = [ style::CSS_MAIN_STYLES, @@ -20,12 +20,12 @@ module Encoders parse style::TOKEN_COLORS end - def get_style styles - cl = @classes[styles.first] + def get_style_for_css_classes css_classes + cl = @styles[css_classes.first] return '' unless cl style = '' - 1.upto styles.size do |offset| - break if style = cl[styles[offset .. -1]] + 1.upto css_classes.size do |offset| + break if style = cl[css_classes[offset .. -1]] end # warn 'Style not found: %p' % [styles] if style.empty? return style @@ -52,8 +52,8 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop - @classes[cl] ||= Hash.new - @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';') + @styles[cl] ||= Hash.new + @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end end end -- cgit v1.2.1 From 2fb89ea3abfd8ee6c4eed8defb805a1abd7187d9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:03:18 +0200 Subject: refactor Scanner a bit (#135) --- lib/coderay/scanner.rb | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 907cf00..80ccc5d 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -184,14 +184,8 @@ module CodeRay options = @options.merge(options) @tokens = options[:tokens] || @tokens || Tokens.new @tokens.scanner = self if @tokens.respond_to? :scanner= - case source - when Array - self.string = self.class.normalize(source.join) - when nil - reset - else - self.string = self.class.normalize(source) - end + + set_string_from_source source begin scan_tokens @tokens, options @@ -261,6 +255,17 @@ module CodeRay def setup # :doc: end + def set_string_from_source source + case source + when Array + self.string = self.class.normalize(source.join) + when nil + reset + else + self.string = self.class.normalize(source) + end + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -277,9 +282,7 @@ module CodeRay @binary_string = nil if defined? @binary_string end - # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller - raise ScanError, <<-EOE % [ + SCAN_ERROR_MESSAGE = <<-MESSAGE ***ERROR in %s: %s (after %d tokens) @@ -297,7 +300,11 @@ surrounding code: ***ERROR*** - EOE + MESSAGE + + # Scanner error with additional status information + def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, tokens.respond_to?(:size) ? tokens.size : 0, -- cgit v1.2.1 From c386e044cff195ae922a0b7625dfa498954bf772 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:15:17 +0200 Subject: refactor Scanner a bit more (#135) --- lib/coderay/scanner.rb | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 80ccc5d..9a5be58 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -182,9 +182,8 @@ module CodeRay # Scan the code and returns all tokens in a Tokens object. def tokenize source = nil, options = {} options = @options.merge(options) - @tokens = options[:tokens] || @tokens || Tokens.new - @tokens.scanner = self if @tokens.respond_to? :scanner= + set_tokens_from_options options set_string_from_source source begin @@ -266,6 +265,11 @@ module CodeRay end end + def set_tokens_from_options options + @tokens = options[:tokens] || @tokens || Tokens.new + @tokens.scanner = self if @tokens.respond_to? :scanner= + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -292,7 +296,7 @@ tokens: current line: %d column: %d pos: %d matched: %p state: %p -bol? = %p, eos? = %p +bol?: %p, eos?: %p surrounding code: %p ~~ %p @@ -303,14 +307,15 @@ surrounding code: MESSAGE # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller + def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, - tokens.respond_to?(:size) ? tokens.size : 0, - tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '', + tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]', + tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]', line, column, pos, - matched, state, bol?, eos?, + matched, state || 'No state given!', + bol?, eos?, binary_string[pos - ambit, ambit], binary_string[pos, ambit], ], backtrace -- cgit v1.2.1 From 65a4b4dfe93e9be0d9f753f063b8f77f8be5af77 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:23:17 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 9a5be58..68451ae 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -289,7 +289,7 @@ module CodeRay SCAN_ERROR_MESSAGE = <<-MESSAGE -***ERROR in %s: %s (after %d tokens) +***ERROR in %s: %s (after %s tokens) tokens: %s @@ -311,8 +311,8 @@ surrounding code: raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, - tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]', - tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]', + tokens_size, + tokens_last(10).map(&:inspect).join("\n"), line, column, pos, matched, state || 'No state given!', bol?, eos?, @@ -321,6 +321,14 @@ surrounding code: ], backtrace end + def tokens_size + tokens.size if tokens.respond_to?(:size) + end + + def tokens_last n + tokens.respond_to?(:last) ? tokens.last(n) : [] + end + # Shorthand for scan_until(/\z/). # This method also avoids a JRuby 1.9 mode bug. def scan_rest -- cgit v1.2.1 From 65b8ab0fda2ac06a2d6109ea96485ab148bab1a4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:32:16 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 68451ae..26bcbf3 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -307,25 +307,27 @@ surrounding code: MESSAGE # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller - raise ScanError, SCAN_ERROR_MESSAGE % [ - File.basename(caller[0]), - msg, - tokens_size, - tokens_last(10).map(&:inspect).join("\n"), + def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace + end + + def raise_inspect_arguments message, tokens, state, ambit + return File.basename(caller[0]), + message, + tokens_size(tokens), + tokens_last(tokens, 10).map(&:inspect).join("\n"), line, column, pos, matched, state || 'No state given!', bol?, eos?, binary_string[pos - ambit, ambit], - binary_string[pos, ambit], - ], backtrace + binary_string[pos, ambit] end - def tokens_size + def tokens_size tokens tokens.size if tokens.respond_to?(:size) end - def tokens_last n + def tokens_last tokens, n tokens.respond_to?(:last) ? tokens.last(n) : [] end -- cgit v1.2.1 From cbf002e7c5e78cee9c2ef7f565b064d9f666bc2b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:39:19 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 26bcbf3..b3f7e17 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -294,9 +294,7 @@ module CodeRay tokens: %s -current line: %d column: %d pos: %d -matched: %p state: %p -bol?: %p, eos?: %p +%s surrounding code: %p ~~ %p @@ -306,21 +304,33 @@ surrounding code: MESSAGE - # Scanner error with additional status information - def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller - raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace - end - def raise_inspect_arguments message, tokens, state, ambit return File.basename(caller[0]), message, tokens_size(tokens), tokens_last(tokens, 10).map(&:inspect).join("\n"), + scanner_state_info(state), + binary_string[pos - ambit, ambit], + binary_string[pos, ambit] + end + + SCANNER_STATE_INFO = <<-INFO +current line: %d column: %d pos: %d +matched: %p state: %p +bol?: %p, eos?: %p + INFO + + def scanner_state_info state + SCANNER_STATE_INFO % [ line, column, pos, matched, state || 'No state given!', bol?, eos?, - binary_string[pos - ambit, ambit], - binary_string[pos, ambit] + ] + end + + # Scanner error with additional status information + def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace end def tokens_size tokens -- cgit v1.2.1 From a69d5d43b79a4f2cf5aaccc50b2275bf0dab4aba Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 23:29:14 +0200 Subject: accept all idents as Ruby 1.9 hash keys (#126) --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c5cf1e2..c282f31 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -96,7 +96,7 @@ module Scanners /#{patterns::METHOD_NAME}/o) kind = patterns::IDENT_KIND[match] - if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/) + if value_expected != :colon_expected && scan(/:(?!:)/) value_expected = true encoder.text_token match, :key encoder.text_token ':', :operator -- cgit v1.2.1 From 33f1a738ea85ea40a5fe760ac2591065f3e29033 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 23:59:47 +0200 Subject: reset string_delimiter when detecting open string --- lib/coderay/scanners/java_script.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 8587b06..9eb0a0a 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -184,6 +184,7 @@ module Scanners elsif match = scan(/ \\ | $ /x) encoder.end_group state encoder.text_token match, :error unless match.empty? + string_delimiter = nil key_expected = value_expected = false state = :initial else -- cgit v1.2.1 From f46a737548583a6cf12cf188bd24f89caa54fde7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 11 Jun 2013 01:09:48 +0200 Subject: set line numbers to word-break: normal to fix #102 --- lib/coderay/styles/alpha.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 1f073b6..81bbe3b 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -39,6 +39,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } color: gray !important; text-decoration: none !important; } +.CodeRay .line-numbers pre { + word-break: normal; +} .CodeRay .line-numbers a:target { color: blue !important; } .CodeRay .line-numbers .highlighted { color: red !important; } .CodeRay .line-numbers .highlighted a { color: red !important; } -- cgit v1.2.1 From b159057934d2c84c0a25c83f6cbe59010345d0a7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 11 Jun 2013 03:29:54 +0200 Subject: tweak CSS tag/id token kind patch from noprompt --- lib/coderay/scanners/css.rb | 4 ++-- lib/coderay/scanners/sass.rb | 6 +++--- lib/coderay/styles/alpha.rb | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 22c6bfc..732f9c5 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -10,7 +10,7 @@ module Scanners :class, :pseudo_class, :tag, :id, :directive, :key, :value, :operator, :color, :float, :string, - :error, :important, + :error, :important, :type, ] # :nodoc: module RE # :nodoc: @@ -99,7 +99,7 @@ module Scanners when :media_before_name if match = scan(RE::Ident) - encoder.text_token match, :tag + encoder.text_token match, :type states[-1] = :media_after_name next end diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 0eb2caa..218ee57 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -38,16 +38,16 @@ module Scanners elsif case states.last when :initial, :media, :sass_inline if match = scan(/(?>#{RE::Ident})(?!\()/ox) - encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :type) + encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) next elsif !value_expected && (match = scan(/\*/)) - encoder.text_token match, :type + encoder.text_token match, :tag next elsif match = scan(RE::Class) encoder.text_token match, :class next elsif match = scan(RE::Id) - encoder.text_token match, :constant + encoder.text_token match, :id next elsif match = scan(RE::PseudoClass) encoder.text_token match, :pseudo_class diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 81bbe3b..a21fbf1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -84,6 +84,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .function .delimiter { color:#024; font-weight:bold } .global-variable { color:#d70 } .hex { color:#02b } +.id { color:#33D; font-weight:bold } .imaginary { color:#f00 } .include { color:#B44; font-weight:bold } .inline { background-color: hsla(0,0%,0%,0.07); color: black } -- cgit v1.2.1 From 2e4e83bf84282d6ce7fbc9eeff2cbe1c79788a9b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 11 Jun 2013 16:30:26 +0200 Subject: tweak Sass scanner: comments, includes, functions --- lib/coderay/scanners/sass.rb | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 218ee57..167051d 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -7,8 +7,6 @@ module Scanners register_for :sass file_extension 'sass' - SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|sprite|unquote|join|round|ceil|floor|nth)/ - STRING_CONTENT_PATTERN = { "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, @@ -26,10 +24,22 @@ module Scanners until eos? - if match = scan(/\s+/) + if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/)) + encoder.text_token self[1], :space if self[1] + encoder.begin_group :comment + encoder.text_token self[2], :delimiter + encoder.text_token self[3], :content if self[3] + if match = scan(/(?:\n+#{self[1]} .*)+/) + encoder.text_token match, :content + end + encoder.end_group :comment + elsif match = scan(/\n|[^\n\S]+\n?/) encoder.text_token match, :space - value_expected = false if match.index(/\n/) - + if match.index(/\n/) + value_expected = false + states.pop if states.last == :include + end + elsif states.last == :sass_inline && (match = scan(/\}/)) encoder.text_token match, :inline_delimiter encoder.end_group :inline @@ -61,7 +71,11 @@ module Scanners elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o) encoder.text_token match, :function next - elsif match = scan(/@media/) + elsif match = scan(/@import\b/) + encoder.text_token match, :directive + states << :include + next + elsif match = scan(/@media\b/) encoder.text_token match, :directive # states.push :media_before_name next @@ -90,13 +104,19 @@ module Scanners encoder.text_token match, :inline_delimiter states.push :sass_inline elsif match = scan(/ \\ | $ /x) - encoder.end_group state + encoder.end_group :string encoder.text_token match, :error unless match.empty? states.pop else raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder end + when :include + if match = scan(/[^\s'",]+/) + encoder.text_token match, :include + next + end + else #:nocov: raise_inspect 'Unknown state', encoder @@ -148,9 +168,6 @@ module Scanners states.push :string end - elsif match = scan(/#{SASS_FUNCTION}/o) - encoder.text_token match, :predefined - elsif match = scan(/#{RE::Function}/o) encoder.begin_group :function start = match[/^[-\w]+\(/] @@ -163,6 +180,9 @@ module Scanners end encoder.end_group :function + elsif match = scan(/[a-z][-a-z_]*(?=\()/o) + encoder.text_token match, :predefined + elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox) encoder.text_token match, :float -- cgit v1.2.1 From abdc3e4a18b664e9d6e20e06913a0aa342f8ca73 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 12 Jun 2013 15:49:44 +0200 Subject: tweak Taskpaper scanner, cleanup doc_string kind --- lib/coderay/encoders/terminal.rb | 2 +- lib/coderay/scanners/taskpaper.rb | 18 ++++++++++-------- lib/coderay/styles/alpha.rb | 3 +-- lib/coderay/token_kinds.rb | 8 ++------ 4 files changed, 14 insertions(+), 17 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index 500e5d8..9894b91 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -37,7 +37,7 @@ module CodeRay :directive => "\e[32m\e[4m", :doc => "\e[46m", :doctype => "\e[1;30m", - :doc_string => "\e[31m\e[4m", + :docstring => "\e[31m\e[4m", :entity => "\e[33m", :error => "\e[1;33m\e[41m", :exception => "\e[1;31m", diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb index c6eb3b3..42670bc 100644 --- a/lib/coderay/scanners/taskpaper.rb +++ b/lib/coderay/scanners/taskpaper.rb @@ -10,17 +10,19 @@ module Scanners def scan_tokens encoder, options until eos? - if match = scan(/^\S.*:.*$/) # project - encoder.text_token(match, :project) - elsif match = scan(/^-.+@done(?:\(.*)?.*$/) # completed task - encoder.text_token(match, :complete) - elsif match = scan(/^-.+$/) # task + if match = scan(/\S.*:.*$/) # project + encoder.text_token(match, :namespace) + elsif match = scan(/-.+@done.*/) # completed task + encoder.text_token(match, :done) + elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task encoder.text_token(match, :plain) - elsif match = scan(/^.+$/) # comment + elsif match = scan(/@due.*/) # comment + encoder.text_token(match, :important) + elsif match = scan(/.+/) # comment encoder.text_token(match, :comment) - elsif match = scan(/\s+/) # space + elsif match = scan(/\s+/) # space encoder.text_token(match, :space) - else # other + else # other encoder.text_token getch, :error end end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 6829686..f57e4a1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -141,8 +141,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .change .change { color: #88f } .head .head { color: #f4f } -.project { color: #707; font-weight: bold } -.complete { text-decoration: line-through; color: gray } +.done { text-decoration: line-through; color: gray } TOKENS end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index f7f19bb..de3a0d0 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -29,7 +29,8 @@ module CodeRay :directive => 'directive', :doc => 'doc', :doctype => 'doctype', - :doc_string => 'doc-string', + :docstring => 'doc-string', + :done => 'done', :entity => 'entity', :error => 'error', :escape => 'escape', @@ -75,9 +76,6 @@ module CodeRay :insert => 'insert', :eyecatcher => 'eyecatcher', - - :project => 'project', - :complete => 'complete', :ident => false, :operator => false, @@ -87,6 +85,4 @@ module CodeRay ) TokenKinds[:method] = TokenKinds[:function] - TokenKinds[:escape] = TokenKinds[:delimiter] - TokenKinds[:docstring] = TokenKinds[:comment] end -- cgit v1.2.1 From 65eeaca6c18bea379b966559084e47d6fb45c75c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 12 Jun 2013 16:19:41 +0200 Subject: add python binary string --- lib/coderay/scanners/python.rb | 4 ++-- lib/coderay/styles/alpha.rb | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index a9492ab..09c8b6e 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -157,12 +157,12 @@ module Scanners encoder.text_token match, :operator elsif match = scan(/(u?r?|b)?("""|"|'''|')/i) + modifiers = self[1] string_delimiter = self[2] - string_type = docstring_coming ? :docstring : :string + string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string) docstring_coming = false if docstring_coming encoder.begin_group string_type string_raw = false - modifiers = self[1] unless modifiers.empty? string_raw = !!modifiers.index(?r) encoder.text_token modifiers, :modifier diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f57e4a1..2ad088a 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -56,7 +56,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .annotation { color:#007 } .attribute-name { color:#b48 } .attribute-value { color:#700 } -.binary { color:#509 } +.binary { color:#549 } +.binary .delimiter { color:#325 } +.binary .char { color:#325 } .char .content { color:#D20 } .char .delimiter { color:#710 } .char { color:#D20 } -- cgit v1.2.1 From 3cc51316b3836c030b7da079f5177a79405700a7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:05:06 +0200 Subject: add taskpaper file extension --- lib/coderay/helpers/file_type.rb | 97 ++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index a5d83ff..6144e8c 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -77,54 +77,55 @@ module CodeRay end TypeFromExt = { - 'c' => :c, - 'cfc' => :xml, - 'cfm' => :xml, - 'clj' => :clojure, - 'css' => :css, - 'diff' => :diff, - 'dpr' => :delphi, - 'erb' => :erb, - 'gemspec' => :ruby, - 'groovy' => :groovy, - 'gvy' => :groovy, - 'h' => :c, - 'haml' => :haml, - 'htm' => :html, - 'html' => :html, - 'html.erb' => :erb, - 'java' => :java, - 'js' => :java_script, - 'json' => :json, - 'mab' => :ruby, - 'pas' => :delphi, - 'patch' => :diff, - 'phtml' => :php, - 'php' => :php, - 'php3' => :php, - 'php4' => :php, - 'php5' => :php, - 'prawn' => :ruby, - 'py' => :python, - 'py3' => :python, - 'pyw' => :python, - 'rake' => :ruby, - 'raydebug' => :raydebug, - 'rb' => :ruby, - 'rbw' => :ruby, - 'rhtml' => :erb, - 'rjs' => :ruby, - 'rpdf' => :ruby, - 'ru' => :ruby, - 'rxml' => :ruby, - 'sass' => :sass, - 'sql' => :sql, - 'tmproj' => :xml, - 'xaml' => :xml, - 'xhtml' => :html, - 'xml' => :xml, - 'yaml' => :yaml, - 'yml' => :yaml, + 'c' => :c, + 'cfc' => :xml, + 'cfm' => :xml, + 'clj' => :clojure, + 'css' => :css, + 'diff' => :diff, + 'dpr' => :delphi, + 'erb' => :erb, + 'gemspec' => :ruby, + 'groovy' => :groovy, + 'gvy' => :groovy, + 'h' => :c, + 'haml' => :haml, + 'htm' => :html, + 'html' => :html, + 'html.erb' => :erb, + 'java' => :java, + 'js' => :java_script, + 'json' => :json, + 'mab' => :ruby, + 'pas' => :delphi, + 'patch' => :diff, + 'phtml' => :php, + 'php' => :php, + 'php3' => :php, + 'php4' => :php, + 'php5' => :php, + 'prawn' => :ruby, + 'py' => :python, + 'py3' => :python, + 'pyw' => :python, + 'rake' => :ruby, + 'raydebug' => :raydebug, + 'rb' => :ruby, + 'rbw' => :ruby, + 'rhtml' => :erb, + 'rjs' => :ruby, + 'rpdf' => :ruby, + 'ru' => :ruby, + 'rxml' => :ruby, + 'sass' => :sass, + 'sql' => :sql, + 'taskpaper' => :taskpaper, + 'tmproj' => :xml, + 'xaml' => :xml, + 'xhtml' => :html, + 'xml' => :xml, + 'yaml' => :yaml, + 'yml' => :yaml, } for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp -- cgit v1.2.1 From 4a85580f76421bb0afb92501119c4dfcfa6540ef Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:12:14 +0200 Subject: tweak Terminal encoder, cleanup token kinds --- lib/coderay/encoders/terminal.rb | 179 ++++++++++++++++++++++----------------- lib/coderay/styles/alpha.rb | 29 +++---- lib/coderay/token_kinds.rb | 136 +++++++++++++++-------------- 3 files changed, 181 insertions(+), 163 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index 9894b91..c7ae014 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -19,105 +19,135 @@ module CodeRay register_for :terminal TOKEN_COLORS = { - :annotation => "\e[35m", - :attribute_name => "\e[33m", + :debug => "\e[1;37;44m", + + :annotation => "\e[34m", + :attribute_name => "\e[35m", :attribute_value => "\e[31m", - :binary => "\e[1;35m", + :binary => { + :self => "\e[31m", + :char => "\e[1;31m", + :delimiter => "\e[1;31m", + }, :char => { - :self => "\e[36m", :delimiter => "\e[1;34m" + :self => "\e[35m", + :delimiter => "\e[1;35m" }, - :class => "\e[1;35m", + :class => "\e[1;35;4m", :class_variable => "\e[36m", :color => "\e[32m", - :comment => "\e[37m", - :complex => "\e[1;34m", - :constant => "\e[1;34m\e[4m", - :decoration => "\e[35m", - :definition => "\e[1;32m", - :directive => "\e[32m\e[4m", - :doc => "\e[46m", - :doctype => "\e[1;30m", - :docstring => "\e[31m\e[4m", - :entity => "\e[33m", - :error => "\e[1;33m\e[41m", + :comment => { + :self => "\e[1;30m", + :char => "\e[37m", + :delimiter => "\e[37m", + }, + :constant => "\e[1;34;4m", + :decorator => "\e[35m", + :definition => "\e[1;33m", + :directive => "\e[33m", + :docstring => "\e[31m", + :doctype => "\e[1;34m", + :done => "\e[1;30;2m", + :entity => "\e[31m", + :error => "\e[1;37;41m", :exception => "\e[1;31m", :float => "\e[1;35m", :function => "\e[1;34m", - :global_variable => "\e[42m", + :global_variable => "\e[1;32m", :hex => "\e[1;36m", - :include => "\e[33m", + :id => "\e[1;34m", + :include => "\e[31m", :integer => "\e[1;34m", - :key => "\e[35m", - :label => "\e[1;15m", + :imaginary => "\e[1;34m", + :important => "\e[1;31m", + :key => { + :self => "\e[35m", + :char => "\e[1;35m", + :delimiter => "\e[1;35m", + }, + :keyword => "\e[32m", + :label => "\e[1;33m", :local_variable => "\e[33m", - :octal => "\e[1;35m", - :operator_name => "\e[1;29m", + :namespace => "\e[1;35m", + :octal => "\e[1;34m", + :predefined => "\e[36m", :predefined_constant => "\e[1;36m", - :predefined_type => "\e[1;30m", - :predefined => "\e[4m\e[1;34m", - :preprocessor => "\e[36m", + :predefined_type => "\e[1;32m", + :preprocessor => "\e[1;36m", :pseudo_class => "\e[1;34m", :regexp => { - :self => "\e[31m", - :content => "\e[31m", - :delimiter => "\e[1;29m", + :self => "\e[35m", + :delimiter => "\e[1;35m", :modifier => "\e[35m", + :char => "\e[1;35m", }, - :reserved => "\e[1;31m", + :reserved => "\e[32m", :shell => { - :self => "\e[42m", - :content => "\e[1;29m", - :delimiter => "\e[37m", + :self => "\e[33m", + :char => "\e[1;33m", + :delimiter => "\e[1;33m", + :escape => "\e[1;33m", }, :string => { - :self => "\e[32m", - :modifier => "\e[1;32m", - :escape => "\e[1;36m", - :delimiter => "\e[1;32m", - :char => "\e[1;36m", + :self => "\e[31m", + :modifier => "\e[1;31m", + :char => "\e[1;35m", + :delimiter => "\e[1;31m", + :escape => "\e[1;31m", + }, + :symbol => { + :self => "\e[33m", + :delimiter => "\e[1;33m", }, - :symbol => "\e[1;32m", - :tag => "\e[1;34m", + :tag => "\e[32m", :type => "\e[1;34m", :value => "\e[36m", - :variable => "\e[1;34m", + :variable => "\e[34m", - :insert => "\e[42m", - :delete => "\e[41m", - :change => "\e[44m", - :head => "\e[45m" + :insert => { + :self => "\e[42m", + :insert => "\e[1;32;42m", + :eyecatcher => "\e[102m", + }, + :delete => { + :self => "\e[41m", + :delete => "\e[1;31;41m", + :eyecatcher => "\e[101m", + }, + :change => { + :self => "\e[44m", + :change => "\e[37;44m", + }, + :head => { + :self => "\e[45m", + :filename => "\e[37;45m" + }, } + TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] - TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex] - TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] = - TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] + TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] protected def setup(options) super @opened = [] - @subcolors = nil + @color_scopes = [TOKEN_COLORS] end public def text_token text, kind - if color = (@subcolors || TOKEN_COLORS)[kind] - if Hash === color - if color[:self] - color = color[:self] - else - @out << text - return - end - end + if color = @color_scopes.last[kind] + color = color[:self] if color.is_a? Hash @out << color - @out << text.gsub("\n", "\e[0m\n" + color) + @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text) @out << "\e[0m" - @out << @subcolors[:self] if @subcolors + if outer_color = @color_scopes.last[:self] + @out << outer_color + end else @out << text end @@ -130,40 +160,33 @@ module CodeRay alias begin_line begin_group def end_group kind - if @opened.empty? - # nothing to close - else - @opened.pop + if @opened.pop + @color_scopes.pop @out << "\e[0m" - @out << open_token(@opened.last) + if outer_color = @color_scopes.last[:self] + @out << outer_color + end end end def end_line kind - if @opened.empty? - # nothing to close - else - @opened.pop - # whole lines to be highlighted, - # eg. added/modified/deleted lines in a diff - @out << (@line_filler ||= "\t" * 100 + "\e[0m") - @out << open_token(@opened.last) - end + @out << (@line_filler ||= "\t" * 100) + end_group kind end private def open_token kind - if color = TOKEN_COLORS[kind] - if Hash === color - @subcolors = color + if color = @color_scopes.last[kind] + if color.is_a? Hash + @color_scopes << color color[:self] else - @subcolors = {} + @color_scopes << @color_scopes.last color end else - @subcolors = nil + @color_scopes << @color_scopes.last '' end end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 2ad088a..9a94091 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -57,26 +57,25 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .attribute-name { color:#b48 } .attribute-value { color:#700 } .binary { color:#549 } -.binary .delimiter { color:#325 } .binary .char { color:#325 } +.binary .delimiter { color:#325 } +.char { color:#D20 } .char .content { color:#D20 } .char .delimiter { color:#710 } -.char { color:#D20 } .class { color:#B06; font-weight:bold } .class-variable { color:#369 } .color { color:#0A0 } .comment { color:#777 } .comment .char { color:#444 } .comment .delimiter { color:#444 } -.complex { color:#A08 } .constant { color:#036; font-weight:bold } .decorator { color:#B0B } .definition { color:#099; font-weight:bold } .delimiter { color:black } .directive { color:#088; font-weight:bold } -.doc { color:#970 } -.doc-string { color:#D42; font-weight:bold } +.docstring { color:#D42; } .doctype { color:#34b } +.done { text-decoration: line-through; color: gray } .entity { color:#800; font-weight:bold } .error { color:#F00; background-color:#FAA } .escape { color:#666 } @@ -87,16 +86,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } -.imaginary { color:#f00 } .include { color:#B44; font-weight:bold } .inline { background-color: hsla(0,0%,0%,0.07); color: black } .inline-delimiter { font-weight: bold; color: #666 } .instance-variable { color:#33B } .integer { color:#00D } +.imaginary { color:#f00 } .important { color:#D00 } +.key { color: #606 } .key .char { color: #60f } .key .delimiter { color: #404 } -.key { color: #606 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } @@ -108,30 +107,30 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .predefined-type { color:#0a5; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } +.regexp { background-color:hsla(300,100%,50%,0.06); } .regexp .content { color:#808 } .regexp .delimiter { color:#404 } .regexp .modifier { color:#C2C } -.regexp { background-color:hsla(300,100%,50%,0.06); } .reserved { color:#080; font-weight:bold } +.shell { background-color:hsla(120,100%,50%,0.06); } .shell .content { color:#2B2 } .shell .delimiter { color:#161 } -.shell { background-color:hsla(120,100%,50%,0.06); } +.string { background-color:hsla(0,100%,50%,0.05); } .string .char { color: #b0b } .string .content { color: #D20 } .string .delimiter { color: #710 } .string .modifier { color: #E40 } -.string { background-color:hsla(0,100%,50%,0.05); } +.symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.symbol { color:#A60 } .tag { color:#070 } .type { color:#339; font-weight:bold } -.value { color: #088; } -.variable { color:#037 } +.value { color: #088 } +.variable { color:#037 } .insert { background: hsla(120,100%,50%,0.12) } .delete { background: hsla(0,100%,50%,0.12) } -.change { color: #bbf; background: #007; } +.change { color: #bbf; background: #007 } .head { color: #f8f; background: #505 } .head .filename { color: white; } @@ -142,8 +141,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .delete .delete { color: #c00; background:transparent; font-weight:bold } .change .change { color: #88f } .head .head { color: #f4f } - -.done { text-decoration: line-through; color: gray } TOKENS end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index de3a0d0..42ea427 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -10,79 +10,77 @@ module CodeRay TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity TokenKinds.update( # :nodoc: - :annotation => 'annotation', - :attribute_name => 'attribute-name', - :attribute_value => 'attribute-value', - :binary => 'bin', - :char => 'char', - :class => 'class', - :class_variable => 'class-variable', - :color => 'color', - :comment => 'comment', - :complex => 'complex', - :constant => 'constant', - :content => 'content', - :debug => 'debug', - :decorator => 'decorator', - :definition => 'definition', - :delimiter => 'delimiter', - :directive => 'directive', - :doc => 'doc', - :doctype => 'doctype', - :docstring => 'doc-string', - :done => 'done', - :entity => 'entity', - :error => 'error', - :escape => 'escape', - :exception => 'exception', - :filename => 'filename', - :float => 'float', - :function => 'function', - :global_variable => 'global-variable', - :hex => 'hex', - :id => 'id', - :imaginary => 'imaginary', - :important => 'important', - :include => 'include', - :inline => 'inline', - :inline_delimiter => 'inline-delimiter', - :instance_variable => 'instance-variable', - :integer => 'integer', - :key => 'key', - :keyword => 'keyword', - :label => 'label', - :local_variable => 'local-variable', - :modifier => 'modifier', - :namespace => 'namespace', - :octal => 'octal', - :predefined => 'predefined', - :predefined_constant => 'predefined-constant', - :predefined_type => 'predefined-type', - :preprocessor => 'preprocessor', - :pseudo_class => 'pseudo-class', - :regexp => 'regexp', - :reserved => 'reserved', - :shell => 'shell', - :string => 'string', - :symbol => 'symbol', - :tag => 'tag', - :type => 'type', - :value => 'value', - :variable => 'variable', + :debug => 'debug', # highlight for debugging (white on blue background) - :change => 'change', - :delete => 'delete', - :head => 'head', - :insert => 'insert', + :annotation => 'annotation', # Groovy, Java + :attribute_name => 'attribute-name', # HTML, CSS + :attribute_value => 'attribute-value', # HTML + :binary => 'binary', # Python, Ruby + :char => 'char', # most scanners, also inside of strings + :class => 'class', # lots of scanners, for different purposes also in CSS + :class_variable => 'class-variable', # Ruby, YAML + :color => 'color', # CSS + :comment => 'comment', # most scanners + :constant => 'constant', # PHP, Ruby + :content => 'content', # inside of strings, most scanners + :decorator => 'decorator', # Python + :definition => 'definition', # CSS + :delimiter => 'delimiter', # inside strings, comments and other types + :directive => 'directive', # lots of scanners + :doctype => 'doctype', # Goorvy, HTML, Ruby, YAML + :docstring => 'docstring', # Python + :done => 'done', # Taskpaper + :entity => 'entity', # HTML + :error => 'error', # invalid token, most scanners + :escape => 'escape', # Ruby (string inline variables like #$foo, #@bar) + :exception => 'exception', # Java, PHP, Python + :filename => 'filename', # Diff + :float => 'float', # most scanners + :function => 'function', # CSS, JavaScript, PHP + :global_variable => 'global-variable', # Ruby, YAML + :hex => 'hex', # hexadecimal number; lots of scanners + :id => 'id', # CSS + :imaginary => 'imaginary', # Python + :important => 'important', # CSS, Taskpaper + :include => 'include', # C, Groovy, Java, Python, Sass + :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :instance_variable => 'instance-variable', # Ruby + :integer => 'integer', # most scanners + :key => 'key', # lots of scanners, used together with :value + :keyword => 'keyword', # reserved word that's actually implemented; most scanners + :label => 'label', # C, PHP + :local_variable => 'local-variable', # local and magic variables; some scanners + :modifier => 'modifier', # used inside on strings; lots of scanners + :namespace => 'namespace', # Clojure, Java, Taskpaper + :octal => 'octal', # lots of scanners + :predefined => 'predefined', # predefined function: lots of scanners + :predefined_constant => 'predefined-constant',# lots of scanners + :predefined_type => 'predefined-type', # C, Java, PHP + :preprocessor => 'preprocessor', # C, Delphi, HTML + :pseudo_class => 'pseudo-class', # CSS + :regexp => 'regexp', # Groovy, JavaScript, Ruby + :reserved => 'reserved', # most scanners + :shell => 'shell', # Ruby + :string => 'string', # most scanners + :symbol => 'symbol', # Clojure, Ruby, YAML + :tag => 'tag', # CSS, HTML + :type => 'type', # CSS, Java, SQL, YAML + :value => 'value', # used together with :key; CSS, JSON, YAML + :variable => 'variable', # Sass, SQL, YAML - :eyecatcher => 'eyecatcher', + :change => 'change', # Diff + :delete => 'delete', # Diff + :head => 'head', # Diff, YAML + :insert => 'insert', # Diff + :eyecatcher => 'eyecatcher', # Diff - :ident => false, - :operator => false, + :ident => false, # almost all scanners + :operator => false, # almost all scanners - :space => false, - :plain => false + :space => false, # almost all scanners + :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] end -- cgit v1.2.1 From df21150ed089f1e3d1d35badde23b2d905a421cd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:35:39 +0200 Subject: speedup/tweak debug encoder --- lib/coderay/encoders/debug.rb | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index c03d3fb..94d52af 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -25,34 +25,37 @@ module Encoders def text_token text, kind raise 'empty token' if $CODERAY_DEBUG && text.empty? + if kind == :space @out << text else - # TODO: Escape ( - text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/) - @out << kind.to_s << '(' << text << ')' + text = text.gsub('\\', '\\\\\\\\') if text.index('\\') + text = text.gsub(')', '\\\\)') if text.index(')') + @out << "#{kind}(#{text})" end end def begin_group kind - @opened << kind - @out << kind.to_s << '<' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}<" end def end_group kind - if @opened.last != kind - puts @out - raise "we are inside #{@opened.inspect}, not #{kind}" - end - @opened.pop + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << '>' end def begin_line kind - @out << kind.to_s << '[' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}[" end def end_line kind + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << ']' end -- cgit v1.2.1 From 88b812274a808f69675445a9265dab1d4e41cc44 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:46:11 +0200 Subject: refactor check functionality to DebugLint encoder --- lib/coderay/encoders/debug.rb | 15 ----------- lib/coderay/encoders/debug_lint.rb | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 lib/coderay/encoders/debug_lint.rb (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 94d52af..61520a1 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -18,14 +18,7 @@ module Encoders FILE_EXTENSION = 'raydebug' - def initialize options = {} - super - @opened = [] - end - def text_token text, kind - raise 'empty token' if $CODERAY_DEBUG && text.empty? - if kind == :space @out << text else @@ -36,26 +29,18 @@ module Encoders end def begin_group kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}<" end def end_group kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << '>' end def begin_line kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}[" end def end_line kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << ']' end diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb new file mode 100644 index 0000000..eeb2a92 --- /dev/null +++ b/lib/coderay/encoders/debug_lint.rb @@ -0,0 +1,55 @@ +module CodeRay +module Encoders + + # = Debug Lint Encoder + # + # Debug encoder with additional checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::Debug + class DebugLint < Debug + + register_for :debug_lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def initialize options = {} + super + @opened = [] + end + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + super + end + + def begin_group kind + @opened << kind + super + end + + def end_group kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + def begin_line kind + @opened << kind + super + end + + def end_line kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + end + +end +end -- cgit v1.2.1 From e423275749744fb2cf768087a07b4c839eaf4734 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 05:33:14 +0200 Subject: remove dump/undump functionality --- lib/coderay/encoders/debug.rb | 1 - lib/coderay/helpers/gzip.rb | 41 ------------------------------------ lib/coderay/tokens.rb | 48 +------------------------------------------ 3 files changed, 1 insertion(+), 89 deletions(-) delete mode 100644 lib/coderay/helpers/gzip.rb (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 61520a1..f4db330 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -9,7 +9,6 @@ module Encoders # # You cannot fully restore the tokens information from the # output, because consecutive :space tokens are merged. - # Use Tokens#dump for caching purposes. # # See also: Scanners::Debug class Debug < Encoder diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb deleted file mode 100644 index 245014a..0000000 --- a/lib/coderay/helpers/gzip.rb +++ /dev/null @@ -1,41 +0,0 @@ -module CodeRay - - # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.) - module GZip - - require 'zlib' - - # The default zipping level. 7 zips good and fast. - DEFAULT_GZIP_LEVEL = 7 - - # Unzips the given string +s+. - # - # Example: - # require 'gzip_simple' - # print GZip.gunzip(File.read('adresses.gz')) - def GZip.gunzip s - Zlib::Inflate.inflate s - end - - # Zips the given string +s+. - # - # Example: - # require 'gzip_simple' - # File.open('adresses.gz', 'w') do |file - # file.write GZip.gzip('Mum: 0123 456 789', 9) - # end - # - # If you provide a +level+, you can control how strong - # the string is compressed: - # - 0: no compression, only convert to gzip format - # - 1: compress fast - # - 7: compress more, but still fast (default) - # - 8: compress more, slower - # - 9: compress best, very slow - def GZip.gzip s, level = DEFAULT_GZIP_LEVEL - Zlib::Deflate.new(level).deflate s, Zlib::FINISH - end - - end - -end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..9142b76 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -1,8 +1,5 @@ module CodeRay - # GZip library for writing and reading token dumps. - autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! # # The Tokens class represents a list of tokens returnd from @@ -45,8 +42,7 @@ module CodeRay # See how small it is? ;) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. + # You can convert it to a webpage, a YAML file, or a .raydebug representation. # # It also allows you to generate tokens directly (without using a scanner), # to load them from a file, and still use any Encoder that CodeRay provides. @@ -157,53 +153,11 @@ module CodeRay parts end - # Dumps the object into a String that can be saved - # in files or databases. - # - # The dump is created with Marshal.dump; - # In addition, it is gzipped using GZip.gzip. - # - # The returned String object includes Undumping - # so it has an #undump method. See Tokens.load. - # - # You can configure the level of compression, - # but the default value 7 should be what you want - # in most cases as it is a good compromise between - # speed and compression rate. - # - # See GZip module. - def dump gzip_level = 7 - dump = Marshal.dump self - dump = GZip.gzip dump, gzip_level - dump.extend Undumping - end - # Return the actual number of tokens. def count size / 2 end - # Include this module to give an object an #undump - # method. - # - # The string returned by Tokens.dump includes Undumping. - module Undumping - # Calls Tokens.load with itself. - def undump - Tokens.load self - end - end - - # Undump the object using Marshal.load, then - # unzip it using GZip.gunzip. - # - # The result is commonly a Tokens object, but - # this is not guaranteed. - def Tokens.load dump - dump = GZip.gunzip dump - @dump = Marshal.load dump - end - alias text_token push def begin_group kind; push :begin_group, kind end def end_group kind; push :end_group, kind end -- cgit v1.2.1 From cb41b00d5673312f4982e914883f9cea95f2ccae Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:12 +0200 Subject: cleanup TODOs, FIXMEs --- lib/coderay/encoders/html/numbering.rb | 2 +- lib/coderay/encoders/statistic.rb | 1 - lib/coderay/token_kinds.rb | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 332145b..5908bf0 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -75,7 +75,7 @@ module Encoders line_number = start output.gsub!(/^.*$\n?/) do |line| line_number_text = bolding.call line_number - indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x) + indent = ' ' * (max_width - line_number.to_s.size) line_number += 1 "#{indent}#{line_number_text}#{line}" end diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index 2315d9e..b2f8b83 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -67,7 +67,6 @@ Token Types (%d): @type_stats['TOTAL'].count += 1 end - # TODO Hierarchy handling def begin_group kind block_token ':begin_group', kind end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 42ea427..9154658 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -44,7 +44,7 @@ module CodeRay :important => 'important', # CSS, Taskpaper :include => 'include', # C, Groovy, Java, Python, Sass :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners - :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why use inline_delimiter? :instance_variable => 'instance-variable', # Ruby :integer => 'integer', # most scanners :key => 'key', # lots of scanners, used together with :value -- cgit v1.2.1 From 462302814dc56c33f26f43516d5ddfc345dcd2a3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:30 +0200 Subject: cleanup Tokens documentation --- lib/coderay/tokens.rb | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..ad59b7f 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,25 +3,23 @@ module CodeRay # GZip library for writing and reading token dumps. autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! + # The Tokens class represents a list of tokens returned from + # a Scanner. It's actually just an Array with a few helper methods. # - # The Tokens class represents a list of tokens returnd from - # a Scanner. - # - # A token is not a special object, just a two-element Array + # A token itself is not a special object, just a two-element Array # consisting of # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # - # A token looks like this: + # It looks like this: # # ['# It looks like this', :comment] # ['3.1415926', :float] # ['$^', :error] # # Some scanners also yield sub-tokens, represented by special - # token actions, namely begin_group and end_group. + # token actions, for example :begin_group and :end_group. # # The Ruby scanner, for example, splits "a string" into: # @@ -33,23 +31,17 @@ module CodeRay # [:end_group, :string] # ] # - # Tokens is the interface between Scanners and Encoders: - # The input is split and saved into a Tokens object. The Encoder - # then builds the output from this object. - # - # Thus, the syntax below becomes clear: - # - # CodeRay.scan('price = 2.59', :ruby).html - # # the Tokens object is here -------^ + # Tokens can be used to save the output of a Scanners in a simple + # Ruby object that can be send to an Encoder later: # - # See how small it is? ;) + # tokens = CodeRay.scan('price = 2.59', :ruby).tokens + # tokens.encode(:html) + # tokens.html + # CodeRay.encoder(:html).encode_tokens(tokens) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. - # - # It also allows you to generate tokens directly (without using a scanner), - # to load them from a file, and still use any Encoder that CodeRay provides. + # You can serialize it to a JSON string and store it in a database, pass it + # around to encode it more than once, send it to other algorithms... class Tokens < Array # The Scanner instance that created the tokens. @@ -58,8 +50,7 @@ module CodeRay # Encode the tokens using encoder. # # encoder can be - # * a symbol like :html oder :statistic - # * an Encoder class + # * a plugin name like :html oder 'statistic' # * an Encoder object # # options are passed to the encoder. -- cgit v1.2.1 From dc9528456008216c265cfb6613cfa459c3c9ffad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:48 +0200 Subject: speedup HTML encoder numbering for Ruby 1.8.7+ --- lib/coderay/encoders/html/numbering.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 5908bf0..a1b9c04 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -26,7 +26,7 @@ module Encoders "#{line}" end else - proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+ + :to_s.to_proc end bold_every = options[:bold_every] -- cgit v1.2.1 From fc21c91d7a89d98785e494ffd41e9f3adbdfed81 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 19 Jun 2013 11:50:04 +0200 Subject: use different key/value heuristic in JSON scanner --- lib/coderay/scanners/json.rb | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 4e0f462..3754a9b 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -14,15 +14,17 @@ module Scanners ESCAPE = / [bfnrt\\"\/] /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: + KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x protected + def setup + @state = :initial + end + # See http://json.org/ for a definition of the JSON lexic/grammar. def scan_tokens encoder, options - - state = :initial - stack = [] - key_expected = false + state = options[:state] || @state until eos? @@ -32,18 +34,11 @@ module Scanners if match = scan(/ \s+ /x) encoder.text_token match, :space elsif match = scan(/"/) - state = key_expected ? :key : :string + state = check(/#{KEY}/o) ? :key : :string encoder.begin_group state encoder.text_token match, :delimiter elsif match = scan(/ [:,\[{\]}] /x) encoder.text_token match, :operator - case match - when ':' then key_expected = false - when ',' then key_expected = true if stack.last == :object - when '{' then stack << :object; key_expected = true - when '[' then stack << :array - when '}', ']' then stack.pop # no error recovery, but works for valid JSON - end elsif match = scan(/ true | false | null /x) encoder.text_token match, :value elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x) @@ -82,6 +77,10 @@ module Scanners end end + if options[:keep_state] + @state = state + end + if [:string, :key].include? state encoder.end_group state end -- cgit v1.2.1 From 5d1812b9d0e1074eb9ec7b24d6ce0f4f7f80619a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:42:30 +0200 Subject: tags should be bold --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 9a94091..f4d07f1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -123,7 +123,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.tag { color:#070 } +.tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } .variable { color:#037 } -- cgit v1.2.1 From 2abfc49bdc9a9f4e86c90aa968c302ca76c20812 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:43:06 +0200 Subject: use JSON to highlight .template files (AWS CF) --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6144e8c..19f27ac 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -120,6 +120,7 @@ module CodeRay 'sass' => :sass, 'sql' => :sql, 'taskpaper' => :taskpaper, + 'template' => :json, # AWS CloudFormation template 'tmproj' => :xml, 'xaml' => :xml, 'xhtml' => :html, -- cgit v1.2.1 From d0d1b129f09dbae8fc25c093c09f72c6396b122a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:53:32 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 2 +- lib/coderay/token_kinds.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c05ffd5..c231423 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index f1696df..9137a49 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :debug => 'debug', # highlight for debugging (white on blue background) -- cgit v1.2.1 From 90adb394e8b4da3a733941fc764c308b34482293 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 00:15:59 +0200 Subject: add lua file type --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 19f27ac..6d4fa92 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -96,6 +96,7 @@ module CodeRay 'java' => :java, 'js' => :java_script, 'json' => :json, + 'lua' => :lua, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, -- cgit v1.2.1 From fbaf55d69c20897655c80e9723973e2f9490484d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:04:53 +0200 Subject: more helpful error messages from DebugLint --- lib/coderay/encoders/debug_lint.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index eeb2a92..0ac89ef 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -35,7 +35,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind super end @@ -45,7 +45,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind super end -- cgit v1.2.1 From c0dc9f2b82b84e6a7511ff9bfe3e80aaa1fef1d1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:05:41 +0200 Subject: move .map styles before .string --- lib/coderay/styles/alpha.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c231423..a869d9e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -99,6 +99,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .namespace { color:#707; font-weight:bold } .octal { color:#40E } .operator { } @@ -123,9 +126,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.map .content { color:#808 } -.map .delimiter { color:#40A} -.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } -- cgit v1.2.1 From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:06:33 +0200 Subject: cleanup Lua scanner, fix end_group(:map) issue - use local instead of instance variables --- lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++++++---------------------- 1 file changed, 267 insertions(+), 257 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 64763dc..3bee275 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -1,265 +1,275 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 -# Scanner for the Lua[http://lua.org] programming lanuage. -# -# The language’s complete syntax is defined in -# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], -# which is what this scanner tries to conform to. -class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner +module CodeRay +module Scanners - register_for :lua - file_extension "lua" - title "Lua" - - # Keywords used in Lua. - KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while - ] - - # Constants set by the Lua core. - PREDEFINED_CONSTANTS = %w[false true nil] - - # The expressions contained in this array are parts of Lua’s `basic' - # library. Although it’s not entirely necessary to load that library, - # it is highly recommended and one would have to provide own implementations - # of some of these expressions if one does not do so. They however aren’t - # keywords, neither are they constants, but nearly predefined, so they - # get tagged as `predefined' rather than anything else. + # Scanner for the Lua[http://lua.org] programming lanuage. # - # This list excludes values of form `_UPPERCASE' because the Lua manual - # requires such identifiers to be reserved by Lua anyway and they are - # highlighted directly accordingly, without the need for specific - # identifiers to be listed here. - PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall - ] - - # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident). - add(KEYWORDS, :keyword). - add(PREDEFINED_CONSTANTS, :predefined_constant). - add(PREDEFINED_EXPRESSIONS, :predefined) - - protected - - # Scanner initialization. - def setup - @state = :initial - @brace_depth = 0 - end - - # CodeRay entry hook. Starts parsing. - def scan_tokens(encoder, options) - @encoder = encoder - @options = options - - until eos? - case state - - when :initial - if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:comment) - @encoder.text_token(match, :delimiter) - @state = :long_comment - - elsif match = scan(/--.*?$/) # --Lua comment - @encoder.text_token(match, :comment) - - elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @state = :long_string - - elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: - @encoder.text_token(match, :label) - - elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua - @encoder.text_token(match, :predefined) - - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) - kind = IDENT_KIND[match] - - # Extra highlighting for entities following certain keywords - if kind == :keyword and match == "function" - @state = :function_expected - elsif kind == :keyword and match == "goto" - @state = :goto_label_expected - elsif kind == :keyword and match == "local" - @state = :local_var_expected + # The language’s complete syntax is defined in + # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], + # which is what this scanner tries to conform to. + class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + state = options[:state] || @state + + until eos? + case state + + when :initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:comment) + encoder.text_token(match, :delimiter) + state = :long_comment + + elsif match = scan(/--.*$/) # --Lua comment + encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + state = :function_expected + elsif kind == :keyword and match == "goto" + state = :goto_label_expected + elsif kind == :keyword and match == "local" + state = :local_var_expected + end + + encoder.text_token(match, kind) + + elsif match = scan(/\{/) # Opening table brace { + encoder.begin_group(:map) + encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + state = :map + + elsif match = scan(/\}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + encoder.text_token(match, :delimiter) + encoder.end_group(:map) + elsif @brace_depth == 0 # Mismatched brace + encoder.text_token(match, :error) + else + @brace_depth -= 1 + encoder.text_token(match, :inline_delimiter) + encoder.end_group(:map) + state = :map + end + + elsif match = scan(/["']/) # String delimiters " and ' + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + @start_delim = match + state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :map scanning state. + state = :map if state == :initial && @brace_depth >= 1 + + when :function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + encoder.text_token(match, :operator) + state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + encoder.text_token(match, :function) + state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + state = :initial + end + + when :goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :label) + state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :local_var_expected + if match = scan(/function/) # local function ... + encoder.text_token(match, :keyword) + state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + encoder.text_token(match, :operator) + elsif match = scan(/\=/) + encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + state = :initial + elsif match = scan(/\n/) + encoder.text_token(match, :space) + state = :initial + elsif match = scan(/\s+/) + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:comment) + state = :initial + + when :long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:string) + state = :initial + + when :string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + encoder.text_token(match, :delimiter) + encoder.end_group(:string) + state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + encoder.end_group(:string) + state = :initial + else + encoder.text_token(getch, :error) + end + + when :map + if match = scan(/[,;]/) + encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + encoder.text_token(match, :key) + encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + encoder.text_token(scan(/\=/), :operator) + state = :initial + elsif match = scan(/\s+/m) + encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + state = :initial + end + else + raise + end + end - - @encoder.text_token(match, kind) - - elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:map) - @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 - @state = :map - - elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 - @encoder.text_token(match, :delimiter) - elsif @brace_depth == 0 # Mismatched brace - @encoder.text_token(match, :error) - else - @brace_depth -= 1 - @encoder.text_token(match, :inline_delimiter) - @state = :map + + if options[:keep_state] + @state = state end - @encoder.end_group(:map) - - elsif match = scan(/["']/) # String delimiters " and ' - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @start_delim = match - @state = :string - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :float) - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :integer) - - elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators - @encoder.text_token(match, :operator) - - elsif match = scan(/\s+/) # Space - @encoder.text_token(match, :space) - - else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. - @encoder.text_token(getch, :error) - end - - # It may be that we’re scanning a full-blown subexpression of a table - # (tables can contain full expressions in parts). - # If this is the case, return to :map scanning state. - @state = :map if @state == :initial && @brace_depth >= 1 - - when :function_expected - if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name - @encoder.text_token(match, :operator) - @state = :initial - elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator - @encoder.text_token(match, :ident) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() - @encoder.text_token(match, :function) - @state = :initial - elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - @state = :initial - end - - when :goto_label_expected - if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :label) - @state = :initial - elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :local_var_expected - if match = scan(/function/) # local function ... - @encoder.text_token(match, :keyword) - @state = :function_expected - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :local_variable) - elsif match = scan(/,/) - @encoder.text_token(match, :operator) - elsif match = scan(/\=/) - @encoder.text_token(match, :operator) - # After encountering the equal sign, arbitrary expressions are - # allowed again, so just return to the main state for further - # parsing. - @state = :initial - elsif match = scan(/\n/) - @encoder.text_token(match, :space) - @state = :initial - elsif match = scan(/\s+/) - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:comment) - @state = :initial - - when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:string) - @state = :initial - - when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) - @encoder.text_token(match, :content) - elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) - @encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) - @encoder.text_token(match, :delimiter) - @encoder.end_group(:string) - @state = :initial - elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings - @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings - @encoder.end_group(:string) - @state = :initial - else - @encoder.text_token(getch, :error) - end - - when :map - if match = scan(/[,;]/) - @encoder.text_token(match, :operator) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) - @encoder.text_token(match, :key) - @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/\=/), :operator) - @state = :initial - elsif match = scan(/\s+/m) - @encoder.text_token(match, :space) - else - # Note this clause doesn’t advance the scan pointer, it’s a kind of - # "retry with other options" (the :initial state then of course - # advances the pointer). - @state = :initial + + encoder end - else - raise + end - - end - - @encoder - end - + +end end -- cgit v1.2.1 From 546b489e4b3856d361a480de5f1f02cb55c7f002 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:09:14 +0200 Subject: fix Tokens documentation --- lib/coderay/tokens.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 54358d4..e7bffce 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,17 +3,16 @@ module CodeRay # The Tokens class represents a list of tokens returned from # a Scanner. It's actually just an Array with a few helper methods. # - # A token itself is not a special object, just a two-element Array - # consisting of + # A token itself is not a special object, just two elements in an Array: # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # # It looks like this: # - # ['# It looks like this', :comment] - # ['3.1415926', :float] - # ['$^', :error] + # ..., '# It looks like this', :comment, ... + # ..., '3.1415926', :float, ... + # ..., '$^', :error, ... # # Some scanners also yield sub-tokens, represented by special # token actions, for example :begin_group and :end_group. @@ -21,11 +20,11 @@ module CodeRay # The Ruby scanner, for example, splits "a string" into: # # [ - # [:begin_group, :string], - # ['"', :delimiter], - # ['a string', :content], - # ['"', :delimiter], - # [:end_group, :string] + # :begin_group, :string, + # '"', :delimiter, + # 'a string', :content, + # '"', :delimiter, + # :end_group, :string # ] # # Tokens can be used to save the output of a Scanners in a simple -- cgit v1.2.1 From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:17:49 +0200 Subject: some more code cleanups before merge --- lib/coderay/scanners/lua.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 3bee275..25bebbe 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -8,17 +8,17 @@ module Scanners # The language’s complete syntax is defined in # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], # which is what this scanner tries to conform to. - class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + class Lua < Scanner register_for :lua - file_extension "lua" - title "Lua" + file_extension 'lua' + title 'Lua' # Keywords used in Lua. KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while + for function goto if in + local not or repeat return + then until while ] # Constants set by the Lua core. @@ -36,10 +36,10 @@ module Scanners # highlighted directly accordingly, without the need for specific # identifiers to be listed here. PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall ] # Automatic token kind selection for normal words. @@ -52,7 +52,7 @@ module Scanners # Scanner initialization. def setup - @state = :initial + @state = :initial @brace_depth = 0 end -- cgit v1.2.1 From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:37:06 +0200 Subject: avoid empty tokens in CSS --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 732f9c5..6c52c8a 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -148,7 +148,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 167051d..65d40b0 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -176,7 +176,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function -- cgit v1.2.1 From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:40:56 +0200 Subject: avoid empty tokens in Raydebug --- lib/coderay/scanners/raydebug.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 7a21354..d39d962 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,11 +1,11 @@ module CodeRay module Scanners - + # = Debug Scanner # # Parses the output of the Encoders::Debug encoder. class Raydebug < Scanner - + register_for :raydebug file_extension 'raydebug' title 'CodeRay Token Dump' @@ -13,11 +13,11 @@ module Scanners protected def scan_tokens encoder, options - + opened_tokens = [] - + until eos? - + if match = scan(/\s+/) encoder.text_token match, :space @@ -26,7 +26,7 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym + encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) @@ -59,8 +59,8 @@ module Scanners encoder end - + end - + end end -- cgit v1.2.1 From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:42:32 +0200 Subject: avoid empty tokens in YAML --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 96f4e93..358a3f1 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -47,7 +47,7 @@ module Scanners when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) encoder.begin_group :string encoder.text_token match, :delimiter - encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx) + encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? encoder.text_token match, :delimiter if match = scan(/"/) encoder.end_group :string next -- cgit v1.2.1 From e7df328cb49fd09a4e5d0f02e877e3270a8703ed Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:53:53 +0200 Subject: DebugLint check open tokens at the end --- lib/coderay/encoders/debug_lint.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 0ac89ef..17a0795 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -19,11 +19,6 @@ module Encoders EmptyToken = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def initialize options = {} - super - @opened = [] - end - def text_token text, kind raise EmptyToken, 'empty token' if text.empty? super @@ -35,7 +30,8 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop super end @@ -45,7 +41,20 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + super + end + + protected + + def setup options + super + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? super end -- cgit v1.2.1 From d6cc5767c9a9bfb4c08bcd5127145b38cb1d29de Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:31 +0200 Subject: HTML encoder shouldn't warn about open tokens --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index b897f5e..20f2409 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -193,7 +193,6 @@ module Encoders def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '' while @opened.pop @last_opened = nil end -- cgit v1.2.1 From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:56 +0200 Subject: close open tokens in Groovy scanner --- lib/coderay/scanners/groovy.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index cf55daf..d04a535 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -36,9 +36,12 @@ module Scanners protected + def setup + @state = :initial + end + def scan_tokens encoder, options - - state = :initial + state = options[:state] || @state inline_block_stack = [] inline_block_paren_depth = nil string_delimiter = nil @@ -246,6 +249,16 @@ module Scanners encoder.end_group state end + if options[:keep_state] + @state = state + end + + until inline_block_stack.empty? + state, = *inline_block_stack.pop + encoder.end_group :inline + encoder.end_group state + end + encoder end -- cgit v1.2.1 From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:55:23 +0200 Subject: close open tokens in PHP scanner --- lib/coderay/scanners/php.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 6c68834..40037f9 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -500,6 +500,10 @@ module Scanners end + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + end + encoder end -- cgit v1.2.1 From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:16 +0200 Subject: remove instance variables from Lua scanner --- lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 25bebbe..185c133 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -59,13 +59,15 @@ module Scanners # CodeRay entry hook. Starts parsing. def scan_tokens(encoder, options) state = options[:state] || @state + brace_depth = @brace_depth + num_equals = nil until eos? case state when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:comment) encoder.text_token(match, :delimiter) state = :long_comment @@ -74,7 +76,7 @@ module Scanners encoder.text_token(match, :comment) elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:string) encoder.text_token(match, :delimiter) state = :long_string @@ -101,19 +103,19 @@ module Scanners elsif match = scan(/\{/) # Opening table brace { encoder.begin_group(:map) - encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 + encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter) + brace_depth += 1 state = :map elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 + if brace_depth == 1 + brace_depth = 0 encoder.text_token(match, :delimiter) encoder.end_group(:map) - elsif @brace_depth == 0 # Mismatched brace + elsif brace_depth == 0 # Mismatched brace encoder.text_token(match, :error) else - @brace_depth -= 1 + brace_depth -= 1 encoder.text_token(match, :inline_delimiter) encoder.end_group(:map) state = :map @@ -122,7 +124,7 @@ module Scanners elsif match = scan(/["']/) # String delimiters " and ' encoder.begin_group(:string) encoder.text_token(match, :delimiter) - @start_delim = match + start_delim = match state = :string # ↓Prefix hex number ←|→ decimal number @@ -146,7 +148,7 @@ module Scanners # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). # If this is the case, return to :map scanning state. - state = :map if state == :initial && @brace_depth >= 1 + state = :map if state == :initial && brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -198,10 +200,10 @@ module Scanners end when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -211,10 +213,10 @@ module Scanners state = :initial when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -224,11 +226,11 @@ module Scanners state = :initial when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) + elsif match = scan(Regexp.compile(start_delim)) encoder.text_token(match, :delimiter) encoder.end_group(:string) state = :initial -- cgit v1.2.1 From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:52 +0200 Subject: close open token groups in Lua scanner --- lib/coderay/scanners/lua.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 185c133..fb1e45a 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -268,6 +268,9 @@ module Scanners @state = state end + encoder.end_group :string if [:string].include? state + brace_depth.times { encoder.end_group :map } + encoder end -- cgit v1.2.1 From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:13:32 +0200 Subject: close open token groups in Sass scanner --- lib/coderay/scanners/sass.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 65d40b0..0ba383f 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -218,6 +218,14 @@ module Scanners @state = states end + while state = states.pop + if state == :sass_inline + encoder.end_group :inline + elsif state == :string + encoder.end_group :string + end + end + encoder end -- cgit v1.2.1 From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:15:55 +0200 Subject: fix Sass regexp modifier --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 0ba383f..e20bebe 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -195,7 +195,7 @@ module Scanners elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + elsif match = scan(/@else if\b|#{RE::AtKeyword}/o) encoder.text_token match, :directive value_expected = true -- cgit v1.2.1 From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:24 +0200 Subject: avoid empty tokens in Diff scanner --- lib/coderay/scanners/diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index af0f755..fd1aed6 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -69,7 +69,7 @@ module Scanners state = :added elsif match = scan(/\\ .*/) encoder.text_token match, :comment - elsif match = scan(/@@(?>[^@\n]*)@@/) + elsif match = scan(/@@(?>[^@\n]+)@@/) content_scanner.state = :initial unless match?(/\n\+/) content_scanner_entry_state = nil if check(/\n|$/) -- cgit v1.2.1 From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:44 +0200 Subject: close correct token groups in Groovy scanner --- lib/coderay/scanners/groovy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index d04a535..c64454f 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -226,7 +226,7 @@ module Scanners encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string encoder.text_token match, :error after_def = value_expected = false state = :initial @@ -246,7 +246,7 @@ module Scanners end if [:multiline_string, :string, :regexp].include? state - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end if options[:keep_state] @@ -256,7 +256,7 @@ module Scanners until inline_block_stack.empty? state, = *inline_block_stack.pop encoder.end_group :inline - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end encoder -- cgit v1.2.1 From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:36:46 +0200 Subject: avoid empty tokens in YAML scanner --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 358a3f1..32c8e2c 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -84,7 +84,7 @@ module Scanners when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) encoder.begin_group :key encoder.text_token match[0,1], :delimiter - encoder.text_token match[1..-2], :content + encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter encoder.end_group :key key_indent = column(pos - match.size) - 1 -- cgit v1.2.1 From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:39:30 +0200 Subject: close open string token groups in SQL scanner --- lib/coderay/scanners/sql.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index b757278..93aeaf3 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -1,8 +1,9 @@ -module CodeRay module Scanners +module CodeRay +module Scanners # by Josh Goebel class SQL < Scanner - + register_for :sql KEYWORDS = %w( @@ -149,6 +150,7 @@ module CodeRay module Scanners string_content = '' end encoder.text_token match, :error unless match.empty? + encoder.end_group :string state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder @@ -171,4 +173,5 @@ module CodeRay module Scanners end -end end \ No newline at end of file +end +end -- cgit v1.2.1 From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:15:10 +0200 Subject: fix unclosed token group in Ruby scanner --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c282f31..80165ca 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -269,7 +269,7 @@ module Scanners end if last_state - state = last_state + state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens last_state = nil end -- cgit v1.2.1 From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:27:43 +0200 Subject: fix token nesting in PHP scanner --- lib/coderay/scanners/php.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 40037f9..7a8d75d 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -265,7 +265,7 @@ module Scanners @html_scanner.tokenize match unless match.empty? end - when :php + when :php, :php_inline if match = scan(/\s+/) encoder.text_token match, :space @@ -332,7 +332,7 @@ module Scanners if states.size == 1 encoder.text_token match, :error else - states.pop + state = states.pop if states.last.is_a?(::Array) delimiter = states.last[1] states[-1] = states.last[0] @@ -340,6 +340,7 @@ module Scanners encoder.end_group :inline else encoder.text_token match, :operator + encoder.end_group :inline if state == :php_inline label_expected = true end end @@ -350,7 +351,14 @@ module Scanners elsif match = scan(RE::PHP_END) encoder.text_token match, :inline_delimiter - states = [:initial] + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + states << :initial elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o) encoder.begin_group :string @@ -400,6 +408,7 @@ module Scanners elsif match = scan(/\\/) encoder.text_token match, :error else + encoder.end_group :string states.pop end @@ -459,7 +468,7 @@ module Scanners encoder.begin_group :inline states[-1] = [states.last, delimiter] delimiter = nil - states.push :php + states.push :php_inline encoder.text_token match, :delimiter else encoder.text_token match, :content @@ -469,6 +478,7 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :content else + encoder.end_group :string states.pop end @@ -502,6 +512,10 @@ module Scanners while state = states.pop encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end end encoder -- cgit v1.2.1 From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:13:15 +0200 Subject: tweak HTML CDATA token kinds --- lib/coderay/scanners/html.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index fcf249a..530ce44 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -1,13 +1,13 @@ module CodeRay module Scanners - + # HTML Scanner # # Alias: +xhtml+ # # See also: Scanners::XML class HTML < Scanner - + register_for :html KINDS_NOT_LOC = [ @@ -100,15 +100,13 @@ module Scanners when :initial if match = scan(//m) - encoder.text_token match[0..-4], :content - encoder.text_token ']]>', :delimiter - else - encoder.text_token scan(/.*/m), :error + encoder.text_token match[0..-4], :plain + encoder.text_token ']]>', :inline_delimiter + elsif match = scan(/.+/) + encoder.text_token match, :error end - encoder.end_group :string elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) -- cgit v1.2.1 From 5b0dc0f35090949d7512a85dadc8cf2871b91aac Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:15:01 +0200 Subject: scan_css in HTML scanner (tags) --- lib/coderay/scanners/html.rb | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..06728e4 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -75,9 +75,14 @@ module Scanners def scan_java_script encoder, code if code && !code.empty? @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true - # encoder.begin_group :inline @java_script_scanner.tokenize code, :tokens => encoder - # encoder.end_group :inline + end + end + + def scan_css encoder, code + if code && !code.empty? + @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true + @css_scanner.tokenize code, :tokens => encoder end end @@ -110,7 +115,7 @@ module Scanners elsif match = scan(/<\/[-\w.:]*>?/m) in_tag = nil encoder.text_token match, :tag - elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m) + elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m) encoder.text_token match, :tag in_tag = self[1] if self[2] @@ -206,19 +211,23 @@ module Scanners when :in_special_tag case in_tag - when 'script' + when 'script', 'style' encoder.text_token match, :space if match = scan(/[ \t]*\n/) if scan(/(\s*)|(.*))/m) code = self[2] || self[4] closing = self[3] encoder.text_token self[1], :comment else - code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/) + code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/) closing = false end unless code.empty? encoder.begin_group :inline - scan_java_script encoder, code + if in_tag == 'script' + scan_java_script encoder, code + else + scan_css encoder, code + end encoder.end_group :inline end encoder.text_token closing, :comment if closing -- cgit v1.2.1 From 7ada74c8f0f77815393e887db83e1d2c36ce7235 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:32:14 +0200 Subject: scan_css in HTML scanner (arguments), change token kind from :inline to :string --- lib/coderay/scanners/html.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 06728e4..f1dfba0 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -33,7 +33,8 @@ module Scanners ) IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil). - add(EVENT_ATTRIBUTES, :script) + add(EVENT_ATTRIBUTES, :script). + add(['style'], :style) ATTR_NAME = /[\w.:-]+/ # :nodoc: TAG_END = /\/?>/ # :nodoc: @@ -79,10 +80,10 @@ module Scanners end end - def scan_css encoder, code + def scan_css encoder, code, state = [:initial] if code && !code.empty? @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true - @css_scanner.tokenize code, :tokens => encoder + @css_scanner.tokenize code, :tokens => encoder, :state => state end end @@ -166,17 +167,21 @@ module Scanners encoder.text_token match, :attribute_value state = :attribute elsif match = scan(/["']/) - if in_attribute == :script - encoder.begin_group :inline - encoder.text_token match, :inline_delimiter + if in_attribute == :script || in_attribute == :style + encoder.begin_group :string + encoder.text_token match, :delimiter if scan(/javascript:[ \t]*/) encoder.text_token matched, :comment end code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/) - scan_java_script encoder, code + if in_attribute == :script + scan_java_script encoder, code + else + scan_css encoder, code, [:block] + end match = scan(/["']/) - encoder.text_token match, :inline_delimiter if match - encoder.end_group :inline + encoder.text_token match, :delimiter if match + encoder.end_group :string state = :attribute in_attribute = nil else -- cgit v1.2.1 From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:33:22 +0200 Subject: fix another CSS empty token issue --- lib/coderay/scanners/css.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 6c52c8a..9ed4618 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -145,10 +145,10 @@ module Scanners start = match[/^\w+\(/] encoder.text_token start, :delimiter if match[-1] == ?) - encoder.text_token match[start.size..-2], :content + encoder.text_token match[start.size..-2], :content if match.size > start.size + 1 encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content if start.size < match.size + encoder.text_token match[start.size..-1], :content if match.size > start.size end encoder.end_group :function -- cgit v1.2.1 From 9fb2fd9fa3ba4b92a440f271adbb9584b236c34e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:45:31 +0200 Subject: tweak :local_variable token color --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index a869d9e..b05ae53 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -98,7 +98,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .key .delimiter { color: #404 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } -.local-variable { color:#963 } +.local-variable { color:#950 } .map .content { color:#808 } .map .delimiter { color:#40A} .map { background-color:hsla(200,100%,50%,0.06); } -- cgit v1.2.1 From 844975b53233b4697d1385548c1295ce18a7be72 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:46:45 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index b05ae53..ff85ecc 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -3,14 +3,14 @@ module Styles # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style - + register_for :alpha - + code_background = 'hsl(0,0%,95%)' numbers_background = 'hsl(180,65%,90%)' border_color = 'silver' normal_color = 'black' - + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; @@ -145,8 +145,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .change .change { color: #88f } .head .head { color: #f4f } TOKENS - + end - + end end -- cgit v1.2.1 From b89caf96d1cfc304c2114d8734ebe8b91337c799 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:53:14 +0200 Subject: fix diff/json token nesting issue --- lib/coderay/scanners/json.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 3754a9b..b09970c 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -26,6 +26,10 @@ module Scanners def scan_tokens encoder, options state = options[:state] || @state + if [:string, :key].include? state + encoder.begin_group state + end + until eos? case state -- cgit v1.2.1 From a3e79c794f0998034b26ff6baa10bc801918be9d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:07:57 +0200 Subject: trying to fix TravisCI error --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f085..4115d09 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.0' unless defined? VERSION end -- cgit v1.2.1 From 416243353e0b4b1c449476b269a50b0f60c13bda Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:15:24 +0200 Subject: trying different ways to load CodeRay version --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4115d09..4b4f085 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' unless defined? VERSION + VERSION = '1.1.0' end -- cgit v1.2.1 From e1e6a6af97ef710cfc02a1380180b772f1c34672 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:23:54 -0600 Subject: additional Ruby files types Ruby seems to have no shortage of these specially named files. --- lib/coderay/helpers/file_type.rb | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d4fa92..6d928d6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -116,7 +116,7 @@ module CodeRay 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, - 'ru' => :ruby, + 'ru' => :ruby, # config.ru 'rxml' => :ruby, 'sass' => :sass, 'sql' => :sql, @@ -132,16 +132,19 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, + 'Guardfile' => :ruby, + 'Vagrantfile' => :ruby, + 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From fd8c81f5c338e4f1448007c9100d15d912fadd27 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:43:54 -0600 Subject: additional types: string, error http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..a66a5e3 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'error', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 004d0c83222113ff732101a3e25785ce0625bfa2 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:14:29 +0200 Subject: whitespace --- lib/coderay/scanners/go.rb | 82 +++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..d4a3598 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,12 +1,12 @@ module CodeRay module Scanners - + # Scanner for Go, copy from c class Go < Scanner - + register_for :go file_extension 'go' - + # http://golang.org/ref/spec#Keywords KEYWORDS = [ 'break', 'default', 'func', 'interface', 'select', @@ -15,7 +15,7 @@ module Scanners 'const', 'fallthrough', 'if', 'range', 'type', 'continue', 'for', 'import', 'return', 'var', ] # :nodoc: - + # http://golang.org/ref/spec#Types PREDEFINED_TYPES = [ 'bool', @@ -26,51 +26,51 @@ module Scanners 'byte', 'rune', 'uint', 'int', 'uintptr', ] # :nodoc: - + PREDEFINED_CONSTANTS = [ 'nil', 'iota', 'true', 'false', ] # :nodoc: - + DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: - + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - - protected - + + protected + def scan_tokens encoder, options - + state = :initial label_expected = true case_expected = false label_expected_before_preproc_line = nil in_preproc_line = false - + until eos? - + case state - + when :initial - + if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space - + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) label_expected = match =~ /[;\{\}]/ if case_expected @@ -78,7 +78,7 @@ module Scanners case_expected = false end encoder.text_token match, :operator - + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) @@ -94,7 +94,7 @@ module Scanners end end encoder.text_token match, kind - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -107,41 +107,41 @@ module Scanners elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment - + elsif match = scan(/#[ \t]*(\w*)/) encoder.text_token match, :preprocessor in_preproc_line = true label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) label_expected = false encoder.text_token match, :char - + elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer - + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) label_expected = false encoder.text_token match, :float - + else encoder.text_token getch, :error - + end - + when :string if match = scan(/[^\\\n"]+/) encoder.text_token match, :content @@ -160,36 +160,36 @@ module Scanners else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end - + when :include_expected if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) encoder.text_token match, :include state = :initial - + elsif match = scan(/\s+/) encoder.text_token match, :space state = :initial if match.index ?\n - + else state = :initial - + end - + else raise_inspect 'Unknown state', encoder - + end - + end - + if state == :string encoder.end_group :string end - + encoder end - + end - + end end -- cgit v1.2.1 From afa2be73c03d033056024354787abd66f2ff7fa0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:05 +0200 Subject: add string as predefined type --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index d4a3598..dbf9595 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 7ef6f7783d788bf656a3c59600cd45cd5b694376 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:22 +0200 Subject: add support for raw strings in Go --- lib/coderay/scanners/go.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index dbf9595..c0c602d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -103,7 +103,14 @@ module Scanners end encoder.text_token match, :delimiter state = :string - + + elsif match = scan(/ ` ([^`]+)? (`)? /x) + encoder.begin_group :shell + encoder.text_token '`', :delimiter + encoder.text_token self[1], :content if self[1] + encoder.text_token self[2], :delimiter if self[2] + encoder.end_group :shell + elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment -- cgit v1.2.1 From a24c207a7e547408de0f6157d495b0dd2e8d5a40 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:36 +0200 Subject: fix empty token in Go scanner --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index c0c602d..76a0820 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -161,7 +161,7 @@ module Scanners encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 4669b7086136647f2eece41a122fa204eb16e1e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:16:47 +0200 Subject: fix label_expected (test case?) --- lib/coderay/scanners/go.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 76a0820..23cbfe6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -64,6 +64,7 @@ module Scanners if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false + case_expected = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space @@ -72,7 +73,6 @@ module Scanners encoder.text_token match, :comment elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) - label_expected = match =~ /[;\{\}]/ if case_expected label_expected = true if match == ':' case_expected = false @@ -83,6 +83,7 @@ module Scanners kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) kind = :label + label_expected = false match << matched else label_expected = false -- cgit v1.2.1 From c0d028010b081020352444078a685ee60bffe209 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:17:14 +0200 Subject: make predefined-type a bit more bright/blue --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index ff85ecc..f4e9d7d 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -107,7 +107,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .operator { } .predefined { color:#369; font-weight:bold } .predefined-constant { color:#069 } -.predefined-type { color:#0a5; font-weight:bold } +.predefined-type { color:#0a8; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } .regexp { background-color:hsla(300,100%,50%,0.06); } -- cgit v1.2.1 From 85275cf21e7d15459abc11fd76606bd7d38fb8b5 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:11:49 -0600 Subject: Go doesn't have a "f" suffix for floats like C. --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 49d24c2..afcfca5 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -141,7 +141,7 @@ module Scanners label_expected = false encoder.text_token match, :integer - elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float -- cgit v1.2.1 From cad9a00e28d61781d5a12a0556be7126eb790725 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:35:26 -0600 Subject: add imaginary numbers to Go scanner --- lib/coderay/scanners/go.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index afcfca5..04504ab 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -128,7 +128,11 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - + + elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + label_expected = false + encoder.text_token match, :imaginary + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex @@ -137,13 +141,13 @@ module Scanners label_expected = false encoder.text_token match, :octal - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) - label_expected = false - encoder.text_token match, :integer - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer else encoder.text_token getch, :error -- cgit v1.2.1 From 4c877bf0eb655cf2c04a0efb24b4e5f83b97e160 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:43:50 -0600 Subject: predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 04504ab..ae50aa4 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,6 +31,11 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: + + PREDEFINED_FUNCTIONS = %w[ + append cap close complex copy delete imag len + make new panic print println real recover + ] # :nodoc: DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? @@ -40,7 +45,8 @@ module Scanners add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). - add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: -- cgit v1.2.1 From dd9ec43a3ea4cfce8eb1e8dd38504a9d8f24df54 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:46:15 -0600 Subject: yup, no C-style directives (auto extern static) --- lib/coderay/scanners/go.rb | 5 ----- 1 file changed, 5 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index ae50aa4..eb06fb0 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -37,14 +37,9 @@ module Scanners make new panic print println real recover ] # :nodoc: - DIRECTIVES = [ - 'go_no_directive', # Seems no directive concept in Go? - ] # :nodoc: - IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). - add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant). add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: -- cgit v1.2.1 From ae1969b1809365c87a9cb485b671364c502236a0 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 13:00:17 -0600 Subject: revert trimming spaces --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d928d6..a0f9fd6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -132,9 +132,9 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, @@ -144,7 +144,7 @@ module CodeRay 'Vagrantfile' => :ruby, 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From dc57601571af8024700991b6a80129285a980e9e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:38 +0200 Subject: add Lint encoder; do we still need DebugLint? --- lib/coderay/encoders/debug_lint.rb | 12 ++++---- lib/coderay/encoders/lint.rb | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/coderay/encoders/lint.rb (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 17a0795..2c14186 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,12 +17,8 @@ module Encoders register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token' if text.empty? super end @@ -30,7 +28,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end @@ -41,7 +39,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 0000000..4601e90 --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,57 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end -- cgit v1.2.1 From 94e4bb3366537def28ed257fcf2b70a634711c9f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 12 Jul 2013 14:57:46 +0200 Subject: don't change value of objects you don't own --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 9ed4618..5977b9c 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -53,7 +53,7 @@ module Scanners end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup value_expected = @value_expected until eos? diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index e20bebe..7ba9bf5 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -19,7 +19,7 @@ module Scanners end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup string_delimiter = nil until eos? @@ -119,7 +119,7 @@ module Scanners else #:nocov: - raise_inspect 'Unknown state', encoder + raise_inspect 'Unknown state: %p' % [states.last], encoder #:nocov: end @@ -215,7 +215,7 @@ module Scanners end if options[:keep_state] - @state = states + @state = states.dup end while state = states.pop -- cgit v1.2.1 From 990ed25fd1bf350dabae4bed031e07ee485beb79 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:20:59 +0200 Subject: split '" string states in Sass scanner (edge case) --- lib/coderay/scanners/sass.rb | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 7ba9bf5..85b4711 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -7,11 +7,6 @@ module Scanners register_for :sass file_extension 'sass' - STRING_CONTENT_PATTERN = { - "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - } - protected def setup @@ -20,7 +15,8 @@ module Scanners def scan_tokens encoder, options states = Array(options[:state] || @state).dup - string_delimiter = nil + + encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring until eos? @@ -91,24 +87,23 @@ module Scanners next end - when :string - if match = scan(STRING_CONTENT_PATTERN[string_delimiter]) + when :sqstring, :dqstring + if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o) encoder.text_token match, :content elsif match = scan(/['"]/) encoder.text_token match, :delimiter encoder.end_group :string - string_delimiter = nil states.pop elsif match = scan(/#\{/) encoder.begin_group :inline encoder.text_token match, :inline_delimiter states.push :sass_inline elsif match = scan(/ \\ | $ /x) - encoder.end_group :string + encoder.end_group states.last encoder.text_token match, :error unless match.empty? states.pop else - raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder + raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder end when :include @@ -157,15 +152,15 @@ module Scanners elsif match = scan(/['"]/) encoder.begin_group :string - string_delimiter = match encoder.text_token match, :delimiter if states.include? :sass_inline - content = scan_until(/(?=#{string_delimiter}|\}|\z)/) + # no nesting, just scan the string until delimiter + content = scan_until(/(?=#{match}|\}|\z)/) encoder.text_token content, :content unless content.empty? - encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/) + encoder.text_token match, :delimiter if scan(/#{match}/) encoder.end_group :string else - states.push :string + states.push match == "'" ? :sqstring : :dqstring end elsif match = scan(/#{RE::Function}/o) @@ -221,7 +216,7 @@ module Scanners while state = states.pop if state == :sass_inline encoder.end_group :inline - elsif state == :string + elsif state == :sqstring || state == :dqstring encoder.end_group :string end end -- cgit v1.2.1 From def7e09db1963368e20bfd53c72532d6a631e0e8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:21:42 +0200 Subject: fix #139: don't scan for :include after eos --- lib/coderay/scanners/sass.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 85b4711..1bbd534 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -209,6 +209,8 @@ module Scanners end + states.pop if states.last == :include + if options[:keep_state] @state = states.dup end -- cgit v1.2.1 From 62a0be9509f8814902a4a97df4ad84913728d059 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:17 +0200 Subject: fix #143 (Sass scanner key vs tag heuristic) --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 1bbd534..e3296b9 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -44,7 +44,7 @@ module Scanners elsif case states.last when :initial, :media, :sass_inline if match = scan(/(?>#{RE::Ident})(?!\()/ox) - encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) + encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag) next elsif !value_expected && (match = scan(/\*/)) encoder.text_token match, :tag -- cgit v1.2.1 From 70c9ba896e1bba5ac727fb6fdfc3ba94510e652d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:41 +0200 Subject: fix CSS scanner for things like "nth-child(2n)" --- lib/coderay/scanners/css.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 5977b9c..55d5239 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -25,7 +25,7 @@ module Scanners HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ - Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/ + Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ -- cgit v1.2.1 From bbe4d72ba785f1bd6fd703d63b096a907da1b09f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:31:34 +0200 Subject: tweak numeral tokens handling (#147) --- lib/coderay/scanners/go.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index eb06fb0..938da9d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,7 +31,7 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: - + PREDEFINED_FUNCTIONS = %w[ append cap close complex copy delete imag len make new panic print println real recover @@ -73,7 +73,7 @@ module Scanners elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + elsif match = scan(/ ?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) if case_expected label_expected = true if match == ':' case_expected = false @@ -129,24 +129,24 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - - elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + + elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/) label_expected = false encoder.text_token match, :imaginary - - elsif match = scan(/0[xX][0-9A-Fa-f]+/) + + elsif match = scan(/-?0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float - - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + + elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer -- cgit v1.2.1 From 6dd14ef018ee4417771504117819121bd4b8520d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:34:10 +0200 Subject: be a bit more graceful with buggy Go strings --- lib/coderay/scanners/go.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 938da9d..5034adc 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -165,9 +165,10 @@ module Scanners label_expected = false elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) encoder.text_token match, :char - elsif match = scan(/ \\ | $ /x) + elsif match = scan(/ \\ /x) + encoder.text_token match, :error + elsif match = scan(/$/) encoder.end_group :string - encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 82864efabda9dc17fa6a28b0740ffc8f58706126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:36:27 +0200 Subject: allow unicode characters in char literals --- lib/coderay/scanners/go.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 5034adc..59473f6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -44,7 +44,7 @@ module Scanners add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected @@ -123,7 +123,7 @@ module Scanners label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox) label_expected = false encoder.text_token match, :char -- cgit v1.2.1 From 59ca07b0d1a1710ab729636ea00de4b638f56110 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Jul 2013 11:17:30 +0200 Subject: =?UTF-8?q?add=20Ruby=202=20syntax:=20%i(=E2=80=A6)=20and=20%I(?= =?UTF-8?q?=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/coderay/scanners/ruby/patterns.rb | 5 ++++- lib/coderay/styles/alpha.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index ed071d2..0b36e13 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -157,13 +157,16 @@ module Scanners yield ]) - FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x + FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x FANCY_STRING_KIND = Hash.new(:string).merge({ + 'i' => :symbol, + 'I' => :symbol, 'r' => :regexp, 's' => :symbol, 'x' => :shell, }) FANCY_STRING_INTERPRETED = Hash.new(true).merge({ + 'i' => false, 'q' => false, 's' => false, 'w' => false, diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f4e9d7d..d304dc4 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -125,7 +125,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .string .modifier { color: #E40 } .symbol { color:#A60 } .symbol .content { color:#A60 } -.symbol .delimiter { color:#630 } +.symbol .delimiter { color:#740 } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } -- cgit v1.2.1 From 5c23a731ca55729fc65630eca3b37a5b1a71e5b1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Jul 2013 18:12:04 +0200 Subject: mark possibly problematic spots with FIXME --- lib/coderay/encoders/html.rb | 2 ++ lib/coderay/encoders/html/css.rb | 2 ++ lib/coderay/helpers/file_type.rb | 1 + lib/coderay/helpers/plugin.rb | 1 + lib/coderay/scanners/debug.rb | 2 ++ lib/coderay/scanners/diff.rb | 1 + lib/coderay/scanners/python.rb | 3 +++ lib/coderay/scanners/raydebug.rb | 2 ++ lib/coderay/scanners/ruby/string_state.rb | 1 + 9 files changed, 15 insertions(+) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 20f2409..6dd231a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,6 +142,7 @@ module Encoders HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ + # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end @@ -284,6 +285,7 @@ module Encoders end def make_span_for_kinds method, hint + # FIXME: cache attack Hash.new do |h, kinds| h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin css_class = css_class_for_kinds(kinds) diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index 164d7f8..de98f0e 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,6 +21,7 @@ module Encoders end def get_style_for_css_classes css_classes + # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -52,6 +53,7 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop + # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5e3a1e7..e8a7b75 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -68,6 +68,7 @@ module CodeRay File.open filename, 'r' do |f| if first_line = f.gets if type = first_line[TypeFromShebang] + # FIXME: cache attack type.to_sym end end diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index d14c5a9..3a38bfd 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -207,6 +207,7 @@ module CodeRay id elsif id.is_a? String if id[/\w+/] == id + # FIXME: cache attack id.downcase.to_sym else raise ArgumentError, "Invalid id given: #{id}" diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 566bfa7..9d10864 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -21,6 +21,7 @@ module Scanners encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) + # FIXME: cache attack kind = self[1].to_sym match = self[2].gsub(/\\(.)/m, '\1') unless TokenKinds.has_key? kind @@ -30,6 +31,7 @@ module Scanners encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) + # FIXME: cache attack kind = self[1].to_sym opened_tokens << kind case self[2] diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index fd1aed6..836fa41 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -21,6 +21,7 @@ module Scanners line_kind = nil state = :initial deleted_lines_count = 0 + # FIXME: cache attack scanners = Hash.new do |h, lang| h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true end diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 09c8b6e..23630f9 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -75,10 +75,12 @@ module Scanners <<=? | >>=? | [<>=]=? | != # comparison and assignment /x # :nodoc: + # FIXME: cache attack STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter| h[delimiter] = Regexp.union delimiter # :nodoc: } + # FIXME: cache attack STRING_CONTENT_REGEXP = Hash.new { |h, delimiter| h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x # :nodoc: } @@ -183,6 +185,7 @@ module Scanners kind = :ident elsif kind == :keyword state = DEF_NEW_STATE[match] + # FIXME: cache attack from_import_state << match.to_sym if state == :include_expected end encoder.text_token match, kind diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index d39d962..ca35de0 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -26,6 +26,7 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] + # FIXME: cache attack encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) @@ -39,6 +40,7 @@ module Scanners else raise 'CodeRay bug: This case should not be reached.' end + # FIXME: cache attack kind = kind.to_sym opened_tokens << kind encoder.begin_group kind diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 2f398d1..fe37d07 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -14,6 +14,7 @@ module Scanners { } ] ].each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with << + # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k # delim = delim.dup # workaround for old Ruby -- cgit v1.2.1 From ea107396fdd72cdbbaf4820d09a87bd879ba7e6c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:43:19 +0200 Subject: check token kinds in Lint encoders --- lib/coderay/encoders/debug_lint.rb | 3 ++- lib/coderay/encoders/lint.rb | 4 +++- lib/coderay/token_kinds.rb | 5 +---- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 2c14186..a4eba2c 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -18,7 +18,8 @@ module Encoders register_for :debug_lint def text_token text, kind - raise Lint::EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb index 4601e90..88c8bd1 100644 --- a/lib/coderay/encoders/lint.rb +++ b/lib/coderay/encoders/lint.rb @@ -17,10 +17,12 @@ module Encoders InvalidTokenStream = Class.new StandardError EmptyToken = Class.new InvalidTokenStream + UnknownTokenKind = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind end def begin_group kind diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 9137a49..5f49d77 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,10 +1,7 @@ module CodeRay # A Hash of all known token kinds and their associated CSS classes. - TokenKinds = Hash.new do |h, k| - warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG - false - end + TokenKinds = Hash.new(false) # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity -- cgit v1.2.1 From 60afd6857c8d0f1c3f9f2d6ca45f01b216d6b4b5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:49:40 +0200 Subject: no attack vector found --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 6dd231a..ee2d91a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,7 +142,6 @@ module Encoders HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ - # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end -- cgit v1.2.1 From 5cd749771379b9832ab1b37936bd98fb7cc80a34 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:09 +0200 Subject: don't dup @span_for_kinds hash keys --- lib/coderay/encoders/html.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ee2d91a..ad5fafc 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -286,7 +286,7 @@ module Encoders def make_span_for_kinds method, hint # FIXME: cache attack Hash.new do |h, kinds| - h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + h[kinds] = begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint -- cgit v1.2.1 From ee30738b0b0615715321aa4f1ed8c7e4025cb411 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:23 +0200 Subject: rename local variable --- lib/coderay/encoders/html.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ad5fafc..81a6ffa 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -310,8 +310,8 @@ module Encoders def break_lines text, style reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '') + @opened.each_with_index do |kind, index| + reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") end -- cgit v1.2.1 From e2546068d0f16fcba15268e740bbb6d9f4f223e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:28:54 +0200 Subject: prevent Symbol attack in Raydebug scanner --- lib/coderay/scanners/raydebug.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index ca35de0..6c1c10f 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,3 +1,5 @@ +require 'set' + module CodeRay module Scanners @@ -12,6 +14,11 @@ module Scanners protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -26,8 +33,13 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - # FIXME: cache attack - encoder.text_token match, kind.to_sym unless match.empty? + unless match.empty? + if @known_token_kinds.include? kind + encoder.text_token match, kind.to_sym + else + encoder.text_token match, :plain + end + end encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) -- cgit v1.2.1 From 8ee1c8deedc58672aa46f311163c2178a70186ce Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:44:01 +0200 Subject: cleanup Plugin, don't use #to_sym anymore --- lib/coderay/helpers/plugin.rb | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 3a38bfd..9a724ff 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -30,7 +30,7 @@ module CodeRay # * a file could not be found # * the requested Plugin is not registered PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError + HostNotFound = Class.new LoadError PLUGIN_HOSTS = [] PLUGIN_HOSTS_BY_ID = {} # dummy hash @@ -49,8 +49,8 @@ module CodeRay def [] id, *args, &blk plugin = validate_id(id) begin - plugin = plugin_hash.[] plugin, *args, &blk - end while plugin.is_a? Symbol + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String plugin end @@ -95,7 +95,7 @@ module CodeRay def map hash for from, to in hash from = validate_id from - to = validate_id to + to = validate_id to plugin_hash[from] = to unless plugin_hash.has_key? from end end @@ -197,23 +197,22 @@ module CodeRay File.join plugin_path, "#{plugin_id}.rb" end - # Converts +id+ to a Symbol if it is a String, - # or returns +id+ if it already is a Symbol. + # Converts +id+ to a valid plugin ID String, or returns +nil+. # # Raises +ArgumentError+ for all other objects, or if the # given String includes non-alphanumeric characters (\W). def validate_id id - if id.is_a? Symbol or id.nil? - id - elsif id.is_a? String + case id + when Symbol + id.to_s + when String if id[/\w+/] == id - # FIXME: cache attack - id.downcase.to_sym + id.downcase else raise ArgumentError, "Invalid id given: #{id}" end else - raise ArgumentError, "String or Symbol expected, but #{id.class} given." + raise ArgumentError, "Symbol or String expected, but #{id.class} given." end end -- cgit v1.2.1 From 6ef7fa4541230442b6e743042648320619ad6859 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:44:55 +0200 Subject: no attack vector found --- lib/coderay/helpers/file_type.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index e8a7b75..5e3a1e7 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -68,7 +68,6 @@ module CodeRay File.open filename, 'r' do |f| if first_line = f.gets if type = first_line[TypeFromShebang] - # FIXME: cache attack type.to_sym end end -- cgit v1.2.1 From 2ab42c7b5e674453fac0320fe0c4a40daf6197e1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:53:41 +0200 Subject: prevent Symbol attack in Debug scanner --- lib/coderay/scanners/debug.rb | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 9d10864..ac12c16 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -1,3 +1,5 @@ +require 'set' + module CodeRay module Scanners @@ -11,6 +13,11 @@ module Scanners protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -21,26 +28,24 @@ module Scanners encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) - # FIXME: cache attack - kind = self[1].to_sym - match = self[2].gsub(/\\(.)/m, '\1') - unless TokenKinds.has_key? kind - kind = :error - match = matched + if @known_token_kinds.include? self[1] + encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym + else + encoder.text_token matched, :error end - encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) - # FIXME: cache attack - kind = self[1].to_sym - opened_tokens << kind - case self[2] - when '<' - encoder.begin_group kind - when '[' - encoder.begin_line kind - else - raise 'CodeRay bug: This case should not be reached.' + if @known_token_kinds.include? self[1] + kind = self[1].to_sym + opened_tokens << kind + case self[2] + when '<' + encoder.begin_group kind + when '[' + encoder.begin_line kind + else + raise 'CodeRay bug: This case should not be reached.' + end end elsif !opened_tokens.empty? && match = scan(/ > /x) -- cgit v1.2.1 From 5d6bee7f5caced1383e6aac427fb356a4788794b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:14:21 +0200 Subject: tweak Debug scanners again, introduce :unknown token kind --- lib/coderay/scanners/debug.rb | 25 ++++++++++++++----------- lib/coderay/scanners/raydebug.rb | 17 ++++++----------- lib/coderay/token_kinds.rb | 3 ++- 3 files changed, 22 insertions(+), 23 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index ac12c16..83ede9a 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -5,7 +5,7 @@ module Scanners # = Debug Scanner # - # Interprets the output of the Encoders::Debug encoder. + # Interprets the output of the Encoders::Debug encoder (basically the inverse function). class Debug < Scanner register_for :debug @@ -31,21 +31,24 @@ module Scanners if @known_token_kinds.include? self[1] encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym else - encoder.text_token matched, :error + encoder.text_token matched, :unknown end elsif match = scan(/ (\w+) ([<\[]) /x) if @known_token_kinds.include? self[1] kind = self[1].to_sym - opened_tokens << kind - case self[2] - when '<' - encoder.begin_group kind - when '[' - encoder.begin_line kind - else - raise 'CodeRay bug: This case should not be reached.' - end + else + kind = :unknown + end + + opened_tokens << kind + case self[2] + when '<' + encoder.begin_group kind + when '[' + encoder.begin_line kind + else + raise 'CodeRay bug: This case should not be reached.' end elsif !opened_tokens.empty? && match = scan(/ > /x) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 6c1c10f..1effdc8 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -3,9 +3,9 @@ require 'set' module CodeRay module Scanners - # = Debug Scanner + # = Raydebug Scanner # - # Parses the output of the Encoders::Debug encoder. + # Highlights the output of the Encoders::Debug encoder. class Raydebug < Scanner register_for :raydebug @@ -43,17 +43,12 @@ module Scanners encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1] - case self[2] - when '<' - encoder.text_token kind, :class - when '[' - encoder.text_token kind, :class + encoder.text_token self[1], :class + if @known_token_kinds.include? self[1] + kind = self[1].to_sym else - raise 'CodeRay bug: This case should not be reached.' + kind = :unknown end - # FIXME: cache attack - kind = kind.to_sym opened_tokens << kind encoder.begin_group kind encoder.text_token self[2], :operator diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 5f49d77..f911862 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -80,5 +80,6 @@ module CodeRay :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:unknown] = TokenKinds[:plain] end -- cgit v1.2.1 From 21d07b305f6293065cf08134cee2c66e727422cf Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:17:47 +0200 Subject: rename protected method in FileType --- lib/coderay/helpers/file_type.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5e3a1e7..7de34d5 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -38,7 +38,7 @@ module CodeRay (TypeFromExt[ext2.downcase] if ext2) || TypeFromName[name] || TypeFromName[name.downcase] - type ||= shebang(filename) if read_shebang + type ||= type_from_shebang(filename) if read_shebang type end @@ -63,7 +63,7 @@ module CodeRay protected - def shebang filename + def type_from_shebang filename return unless File.exist? filename File.open filename, 'r' do |f| if first_line = f.gets -- cgit v1.2.1 From 368e053880819edc74fdcef38f38b5fd4806a3f4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:18:49 +0200 Subject: FileType should guard against attacks here --- lib/coderay/scanners/diff.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index 836fa41..fd1aed6 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -21,7 +21,6 @@ module Scanners line_kind = nil state = :initial deleted_lines_count = 0 - # FIXME: cache attack scanners = Hash.new do |h, lang| h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true end -- cgit v1.2.1 From af04107b8b370452a17fa54e8ea0e8adc8b376b0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:19:57 +0200 Subject: no attack vector, there are only 4 cases --- lib/coderay/scanners/python.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 23630f9..05e1f5f 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -75,12 +75,10 @@ module Scanners <<=? | >>=? | [<>=]=? | != # comparison and assignment /x # :nodoc: - # FIXME: cache attack STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter| h[delimiter] = Regexp.union delimiter # :nodoc: } - # FIXME: cache attack STRING_CONTENT_REGEXP = Hash.new { |h, delimiter| h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x # :nodoc: } -- cgit v1.2.1 From e9140073f4dcba5c022a2ad40a1b935a07a6b4c3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:21:45 +0200 Subject: no attack vector, there are only 2 cases --- lib/coderay/scanners/python.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 05e1f5f..09c8b6e 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -183,7 +183,6 @@ module Scanners kind = :ident elsif kind == :keyword state = DEF_NEW_STATE[match] - # FIXME: cache attack from_import_state << match.to_sym if state == :include_expected end encoder.text_token match, kind -- cgit v1.2.1 From c3c70e0b3497939dbfb1958a0764f4fd18c05a48 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:31:35 +0200 Subject: cleanup --- lib/coderay/scanners/ruby/string_state.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index fe37d07..bcc0507 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -17,7 +17,6 @@ module Scanners # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k - # delim = delim.dup # workaround for old Ruby delim_pattern = Regexp.escape(delim) if closing_paren = CLOSING_PAREN[delim] delim_pattern << Regexp.escape(closing_paren) -- cgit v1.2.1 From 65983f38eaed758a9901adf9e4e8c4be3e3a6123 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:41:55 +0200 Subject: avoid cache attack in Ruby scanner (eg. using Unicode-delimited Fancy Strings) --- lib/coderay/scanners/ruby/string_state.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index bcc0507..28ddd6c 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -14,7 +14,6 @@ module Scanners { } ] ].each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with << - # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k delim_pattern = Regexp.escape(delim) @@ -29,12 +28,13 @@ module Scanners # '| [|?*+(){}\[\].^$]' # end - h[k] = - if interpreted && delim != '#' - / (?= [#{delim_pattern}] | \# [{$@] ) /mx - else - / (?= [#{delim_pattern}] ) /mx - end + if interpreted && delim != '#' + / (?= [#{delim_pattern}] | \# [{$@] ) /mx + else + / (?= [#{delim_pattern}] ) /mx + end.tap do |pattern| + h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256 + end end def initialize kind, interpreted, delim, heredoc = false -- cgit v1.2.1 From 05f5a0e270ce2cde4ff242634033c902c58f13ea Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:45:33 +0200 Subject: no cache attacks possible, static input (CSS) --- lib/coderay/encoders/html/css.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index de98f0e..164d7f8 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,7 +21,6 @@ module Encoders end def get_style_for_css_classes css_classes - # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -53,7 +52,6 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop - # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end -- cgit v1.2.1 From ee992427810a1cf88b53c12ccf7fda91a30ab33e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:58:07 +0200 Subject: limit HTML encoder span_for_kinds cache size --- lib/coderay/encoders/html.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 81a6ffa..ffde5d2 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -284,9 +284,8 @@ module Encoders end def make_span_for_kinds method, hint - # FIXME: cache attack Hash.new do |h, kinds| - h[kinds] = begin + begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint @@ -298,6 +297,9 @@ module Encoders "" end end + end.tap do |span| + h.clear if h.size >= 100 + h[kinds] = span end end end -- cgit v1.2.1 From a31b36683834f39c1581add498cce0b016f20fb5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 30 Aug 2013 16:22:19 +0200 Subject: fix coderay -HTML option --- lib/coderay/encoders/html.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ffde5d2..d2ebb5a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -197,13 +197,15 @@ module Encoders @last_opened = nil end - @out.extend Output - @out.css = @css - if options[:line_numbers] - Numbering.number! @out, options[:line_numbers], options + if @out.respond_to? :to_str + @out.extend Output + @out.css = @css + if options[:line_numbers] + Numbering.number! @out, options[:line_numbers], options + end + @out.wrap! options[:wrap] + @out.apply_title! options[:title] end - @out.wrap! options[:wrap] - @out.apply_title! options[:title] if defined?(@real_out) && @real_out @real_out << @out -- cgit v1.2.1 From a48037b85a12228431b32103786456f36beb355f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 1 Sep 2013 01:01:35 +0200 Subject: final cleanup --- lib/coderay/scanners/go.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 59473f6..99fdd63 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,7 +1,6 @@ module CodeRay module Scanners - # Scanner for Go, copy from c class Go < Scanner register_for :go -- cgit v1.2.1