Lexer, fixes and tests

2 files changed, 129 insertions, 15 deletions

diff --git a/lib/lace/lex.lua b/lib/lace/lex.lua
index c768d08..07277a5 100644
--- a/lib/lace/lex.lua
+++ b/lib/lace/lex.lua
@@ -54,8 +54,11 @@ local function lex_one_line(line)
 	 elseif c == " " or c == "\t" then
 	    -- A space (or tab) and not quoting, so clear the accumulator
 	    if acc ~= "" then
-	       r[#r+1] = { spos, acc }
+	       r[#r+1] = { spos, acc, pos = spos, str = acc }
 	       spos = cpos + 1
+	    elseif cpos == spos then
+	       -- Increment the start position since we've not found a word yet
+	       spos = spos + 1
 	    end
 	    acc = ""
 	 else
@@ -64,7 +67,7 @@ local function lex_one_line(line)
       end
    end
    if acc ~= "" then
-      r[#r+1] = { spos, acc }
+      r[#r+1] = { spos, acc, pos = spos, str = acc }
    end
 
    local warnings = {}
@@ -86,25 +89,28 @@ local function lex_a_ruleset(ruleset, sourcename)
    local ret = { source = sourcename, lines = lines }
    local n = 1
    local warn
+   if ruleset:match("[^\n]$") then
+      ruleset = ruleset .. "\n"
+   end
    for oneline in ruleset:gmatch("([^\n]*)\n") do
       local linetab = { original = oneline }
-      if linetab:find("^[ \t]*#") or
-	 linetab:find("^[ \t]*//") or
-	 linetab:find("^[ \t]*%-%-") then
+      if oneline:match("^[ \t]*#") or
+	 oneline:match("^[ \t]*//") or
+	 oneline:match("^[ \t]*%-%-") then
 	 linetab.type = "comment"
-      elseif linetab:find("^[ \t]*$") then
+      elseif oneline:match("^[ \t]*$") then
 	 linetab.type = "whitespace"
       else
 	 linetab.type = "rule"
 	 linetab.content, warn = lex_one_line(oneline)
-	 if #warn then
+	 if #warn > 0 then
 	    linetab.warnings = warn
 	 end
       end
       lines[n] = linetab
       n = n + 1
    end
-   return lines
+   return ret
 end
 
 local function lex_a_file(filename)
diff --git a/test/test-lace.lex.lua b/test/test-lace.lex.lua
index e0119e4..c583d58 100644
--- a/test/test-lace.lex.lua
+++ b/test/test-lace.lex.lua
@@ -13,22 +13,128 @@ local luacov = require 'luacov'
 
 local lex = require 'lace.lex'
 
-local suite = {}
+local testnames = {}
 
-function suite.test_01_empty_string()
+local function add_test(suite, name, value)
+   rawset(suite, name, value)
+   testnames[#testnames+1] = name
 end
 
-local testnames = {}
-for k in pairs(suite) do
-   testnames[#testnames+1] = k
+local suite = setmetatable({}, {__newindex = add_test})
+
+function suite.empty_string()
+   local content = assert(lex.string("", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 0, "There are lines provided, despite source being empty")
+end
+
+function suite.single_cmd_string()
+   local content = assert(lex.string("hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+end
+
+function suite.single_cmd_two_words_string()
+   local content = assert(lex.string("hello world", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 2, "The line should have 2 words")
+   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+   assert(content.lines[1].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].str == "world", "The word is 'world'")
+end
+
+function suite.two_cmds_two_words_string()
+   local content = assert(lex.string("hello world\nworld hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 2, "There should have been two lines")
+   assert(#content.lines[1].content == 2, "The line should have 2 words")
+   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word starts is 'hello'")
+   assert(content.lines[1].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].str == "world", "The word is 'hello'")
+   assert(#content.lines[2].content == 2, "The line should have 2 words")
+   assert(content.lines[2].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[2].content[1].str == "world", "The word is 'word'")
+   assert(content.lines[2].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[2].content[2].str == "hello", "The word is 'hello'")
+end
+
+function suite.one_hash_comment()
+   local content = assert(lex.string("# Hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(content.lines[1].type == "comment", "The line should be a comment")
+end
+
+function suite.one_slashes_comment()
+   local content = assert(lex.string("// Hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(content.lines[1].type == "comment", "The line should be a comment")
+end
+
+function suite.one_dashes_comment()
+   local content = assert(lex.string("-- Hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(content.lines[1].type == "comment", "The line should be a comment")
+end
+
+function suite.pure_whitespace()
+   local content = assert(lex.string(" ", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(content.lines[1].type == "whitespace", "The line should be whitespace")
+end
+
+function suite.whitespace_then_comment()
+   local content = assert(lex.string(" -- Fish", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(content.lines[1].type == "comment", "The line should be whitespace")
+end
+
+function suite.whitespace_then_command()
+   local content = assert(lex.string("   hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].pos == 4, "The word starts at the fourth character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+end
+
+function suite.whitespace_in_command()
+   local content = assert(lex.string("hello   world", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines), "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 2, "The line should have 2 words")
+   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+   assert(content.lines[1].content[2].pos == 9, "The word starts at the ninth character")
+   assert(content.lines[1].content[2].str == "world", "The word is 'world'")
 end
-table.sort(testnames)
 
 local count_ok = 0
 for _, testname in ipairs(testnames) do
+   print("Run: " .. testname)
    local ok, err = xpcall(suite[testname], debug.traceback)
    if not ok then
-      print(testname .. ":")
       print(err)
       print()
    else
@@ -37,3 +143,5 @@ for _, testname in ipairs(testnames) do
 end
 
 print("Lex: " .. tostring(count_ok) .. "/" .. tostring(#testnames) .. " OK")
+
+os.exit(count_ok == #testnames and 0 or 1)