More lexer testing

2 files changed, 162 insertions, 21 deletions

diff --git a/test/test-lace.lex.lua b/test/test-lace.lex.lua
index c583d58..8f89bf3 100644
--- a/test/test-lace.lex.lua
+++ b/test/test-lace.lex.lua
@@ -25,53 +25,59 @@ local suite = setmetatable({}, {__newindex = add_test})
 function suite.empty_string()
    local content = assert(lex.string("", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 0, "There are lines provided, despite source being empty")
 end
 
 function suite.single_cmd_string()
    local content = assert(lex.string("hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(#content.lines[1].content == 1, "The line should have 1 word")
-   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].epos == 5, "The word ends at the fifth character")
    assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+   assert(type(content.lines[1].warnings) == "nil", "There should be no warnings")
 end
 
 function suite.single_cmd_two_words_string()
    local content = assert(lex.string("hello world", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(#content.lines[1].content == 2, "The line should have 2 words")
-   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].epos == 5, "The word ends at the fifth character")
    assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
-   assert(content.lines[1].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].spos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].epos == 11, "The word ends at the eleventh character")
    assert(content.lines[1].content[2].str == "world", "The word is 'world'")
 end
 
 function suite.two_cmds_two_words_string()
    local content = assert(lex.string("hello world\nworld hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 2, "There should have been two lines")
    assert(#content.lines[1].content == 2, "The line should have 2 words")
-   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
    assert(content.lines[1].content[1].str == "hello", "The word starts is 'hello'")
-   assert(content.lines[1].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].spos == 7, "The word starts at the seventh character")
+   assert(content.lines[1].content[2].epos == 11, "The word ends at the eleventh character")
    assert(content.lines[1].content[2].str == "world", "The word is 'hello'")
    assert(#content.lines[2].content == 2, "The line should have 2 words")
-   assert(content.lines[2].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[2].content[1].spos == 1, "The word starts at the first character")
    assert(content.lines[2].content[1].str == "world", "The word is 'word'")
-   assert(content.lines[2].content[2].pos == 7, "The word starts at the seventh character")
+   assert(content.lines[2].content[2].spos == 7, "The word starts at the seventh character")
+   assert(content.lines[2].content[2].epos == 11, "The word ends at the eleventh character")
    assert(content.lines[2].content[2].str == "hello", "The word is 'hello'")
 end
 
 function suite.one_hash_comment()
    local content = assert(lex.string("# Hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(content.lines[1].type == "comment", "The line should be a comment")
 end
@@ -79,7 +85,7 @@ end
 function suite.one_slashes_comment()
    local content = assert(lex.string("// Hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(content.lines[1].type == "comment", "The line should be a comment")
 end
@@ -87,7 +93,7 @@ end
 function suite.one_dashes_comment()
    local content = assert(lex.string("-- Hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(content.lines[1].type == "comment", "The line should be a comment")
 end
@@ -95,7 +101,7 @@ end
 function suite.pure_whitespace()
    local content = assert(lex.string(" ", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(content.lines[1].type == "whitespace", "The line should be whitespace")
 end
@@ -103,7 +109,7 @@ end
 function suite.whitespace_then_comment()
    local content = assert(lex.string(" -- Fish", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(content.lines[1].type == "comment", "The line should be whitespace")
 end
@@ -111,25 +117,155 @@ end
 function suite.whitespace_then_command()
    local content = assert(lex.string("   hello", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(#content.lines[1].content == 1, "The line should have 1 word")
-   assert(content.lines[1].content[1].pos == 4, "The word starts at the fourth character")
+   assert(content.lines[1].content[1].spos == 4, "The word starts at the fourth character")
+   assert(content.lines[1].content[1].epos == 8, "The word ends at the ninth character")
    assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
 end
 
 function suite.whitespace_in_command()
    local content = assert(lex.string("hello   world", "SRC"))
    assert(content.source == "SRC", "Source name not propagated")
-   assert(type(content.lines), "Lines is not a table")
+   assert(type(content.lines) == "table", "Lines is not a table")
    assert(#content.lines == 1, "There should have been one line")
    assert(#content.lines[1].content == 2, "The line should have 2 words")
-   assert(content.lines[1].content[1].pos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
    assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
-   assert(content.lines[1].content[2].pos == 9, "The word starts at the ninth character")
+   assert(content.lines[1].content[2].spos == 9, "The word starts at the ninth character")
    assert(content.lines[1].content[2].str == "world", "The word is 'world'")
 end
 
+function suite.single_quoted_word()
+   local content = assert(lex.string("'hello'", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].epos == 7, "The word ends at the seventh character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+end
+
+function suite.double_quoted_word()
+   local content = assert(lex.string('"hello"', "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+end
+
+function suite.escape_outside_quotes()
+   local content = assert(lex.string("\\this", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "this", "The word is 'this'")
+end
+
+function suite.escape_inside_normal()
+   local content = assert(lex.string("'hell\\o'", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is 'hello'")
+end
+
+function suite.escape_inside_quotetype()
+   local content = assert(lex.string("'hello\\''", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello'", "The word is \"hello'\"")
+end
+
+function suite.escape_inside_tab()
+   local content = assert(lex.string("'hello\\t'", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello\t", "The word is \"hello\\t\"")
+end
+
+function suite.escape_inside_newline()
+   local content = assert(lex.string("'hello\\n'", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello\n", "The word is \"hello\\n\"")
+end
+
+function suite.escape_outside_unused()
+   local content = assert(lex.string("hello\\", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is \"hello\"")
+   assert(type(content.lines[1].warnings) == "table", "There should be a warning")
+   assert(#content.lines[1].warnings == 1, "There should be one warning")
+   assert(content.lines[1].warnings[1]:find("escape"), "The warning should be about the escape")
+end
+
+function suite.unclosed_quote()
+   local content = assert(lex.string("'hello", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is \"hello\"")
+   assert(type(content.lines[1].warnings) == "table", "There should be a warning")
+   assert(#content.lines[1].warnings == 1, "There should be one warning")
+   assert(content.lines[1].warnings[1]:find("quoted"), "The warning should be about the unclosed quotes")
+end
+
+function suite.escape_inside_unclosed_unused()
+   local content = assert(lex.string("'hello\\", "SRC"))
+   assert(content.source == "SRC", "Source name not propagated")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 1, "There should have been one line")
+   assert(#content.lines[1].content == 1, "The line should have 1 word")
+   assert(content.lines[1].content[1].spos == 1, "The word starts at the first character")
+   assert(content.lines[1].content[1].str == "hello", "The word is \"hello\"")
+   assert(type(content.lines[1].warnings) == "table", "There should be a warning")
+   assert(#content.lines[1].warnings == 2, "There should be two warnings")
+   assert(content.lines[1].warnings[1]:find("quoted"), "The warning should be about the unclosed quotes")
+   assert(content.lines[1].warnings[2]:find("escape"), "The warning should be about the escape")
+end
+
+function suite.load_unknown_file()
+   local ok, msg = lex.file("test/NOT-PRESENT")
+   assert(not ok, "Managed to lex a non-present file?!")
+   assert(msg:match 'Unable to open', "Error doesn't seem right")
+end
+
+function suite.load_known_file()
+   local content = assert(lex.file("test/test-lace.lex.rules"))
+   assert(content.source == "@test/test-lace.lex.rules")
+   assert(type(content.lines) == "table", "Lines is not a table")
+   assert(#content.lines == 5, "There should have been five lines")
+   assert(content.lines[1].type == "comment", "Line 1 should be a comment")
+   assert(content.lines[2].type == "comment", "Line 2 should be a comment")
+   assert(content.lines[3].type == "comment", "Line 3 should be a comment")
+   assert(content.lines[4].type == "whitespace", "Line 3 should be whitespace")
+   assert(content.lines[5].type == "rule", "Line 3 should be a rule")
+end
+
 local count_ok = 0
 for _, testname in ipairs(testnames) do
    print("Run: " .. testname)
diff --git a/test/test-lace.lex.rules b/test/test-lace.lex.rules
new file mode 100644
index 0000000..c3acc51
--- /dev/null
+++ b/test/test-lace.lex.rules
@@ -0,0 +1,5 @@
+-- Trivial ruleset should total 5 lines
+// The first three are comments
+# The fourth is whitespace and the fifth is a statement of 2 words
+
+   deny "I hate world"