From 423c44a451db7e5f63147b1c1519661d745fc43a Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 18 Jan 2021 12:01:21 -0500 Subject: Matlab class properties (#1466) * WIP: Add failing test for a matlab class with properties. * Add some missing keywords * Add leading \s* matchers to things above the command form regex, as it tends to swallow keywords otherwise. * Add support for the special 'properties' block syntax. * Fix apparent infinite loop when given garbage input. * Use includes to clean up some of my copypasta. * Fix negative lookahead when there's more than one space between operators. * Use Whitespace not Text for spaces; combine adjacent whitespace. * Add support for declarative property constraints. --- tests/test_matlab.py | 171 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 137 insertions(+), 34 deletions(-) (limited to 'tests') diff --git a/tests/test_matlab.py b/tests/test_matlab.py index 945a3434..b375c8da 100644 --- a/tests/test_matlab.py +++ b/tests/test_matlab.py @@ -36,7 +36,7 @@ def test_single_line(lexer): (Token.Literal.Number.Integer, '101325'), (Token.Punctuation, ')'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -56,14 +56,14 @@ def test_line_continuation(lexer): (Token.Literal.Number.Integer, '300'), (Token.Punctuation, ','), (Token.Keyword, '...'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), (Token.Literal.String, "'"), (Token.Literal.String, "P'"), (Token.Punctuation, ','), (Token.Literal.Number.Integer, '101325'), (Token.Punctuation, ')'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -73,37 +73,29 @@ def test_keywords_ended_by_newline(lexer): fragment = "if x > 100\n disp('x > 100')\nelse\n disp('x < 100')\nend\n" tokens = [ (Token.Keyword, 'if'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Name, 'x'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Operator, '>'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Literal.Number.Integer, '100'), - (Token.Text, '\n'), - (Token.Text, ' '), - (Token.Text, ' '), - (Token.Text, ' '), - (Token.Text, ' '), + (Token.Text.Whitespace, '\n '), (Token.Name.Builtin, 'disp'), (Token.Punctuation, '('), (Token.Literal.String, "'"), (Token.Literal.String, "x > 100'"), (Token.Punctuation, ')'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), (Token.Keyword, 'else'), - (Token.Text, '\n'), - (Token.Text, ' '), - (Token.Text, ' '), - (Token.Text, ' '), - (Token.Text, ' '), + (Token.Text.Whitespace, '\n '), (Token.Name.Builtin, 'disp'), (Token.Punctuation, '('), (Token.Literal.String, "'"), (Token.Literal.String, "x < 100'"), (Token.Punctuation, ')'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), (Token.Keyword, 'end'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -123,14 +115,14 @@ def test_comment_after_continuation(lexer): (Token.Punctuation, ','), (Token.Keyword, '...'), (Token.Comment, ' a comment'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), (Token.Literal.String, "'"), (Token.Literal.String, "P'"), (Token.Punctuation, ','), (Token.Literal.Number.Integer, '101325'), (Token.Punctuation, ')'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -142,13 +134,12 @@ def test_multiple_spaces_variable_assignment(lexer): fragment = 'x = 100;\n' tokens = [ (Token.Name, 'x'), - (Token.Text, ' '), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Punctuation, '='), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Literal.Number.Integer, '100'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -160,13 +151,12 @@ def test_operator_multiple_space(lexer): fragment = 'x > 100;\n' tokens = [ (Token.Name, 'x'), - (Token.Text, ' '), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Operator, '>'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Literal.Number.Integer, '100'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -176,12 +166,12 @@ def test_one_space_assignment(lexer): fragment = 'x = 100;\n' tokens = [ (Token.Name, 'x'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Punctuation, '='), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Literal.Number.Integer, '100'), (Token.Punctuation, ';'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens @@ -195,8 +185,121 @@ def test_command_mode(lexer): fragment = 'help sin\n' tokens = [ (Token.Name, 'help'), - (Token.Text, ' '), + (Token.Text.Whitespace, ' '), (Token.Literal.String, 'sin'), - (Token.Text, '\n'), + (Token.Text.Whitespace, '\n'), ] assert list(lexer.get_tokens(fragment)) == tokens + + + +MATLAB_SAMPLE_CLASS = """ +classdef Name < dynamicprops + properties + % i am a comment + name1 + name2 + end + properties (Constant = true, SetAccess = protected) + % i too am a comment + matrix = [0, 1, 2]; + string = 'i am a string' + end + methods + % i am also a comment + function self = Name() + % i am a comment inside a constructor + end + end +end +""".strip() + +def test_classes_with_properties(lexer): + whitespace = Token.Text.Whitespace + tokens = [ + (Token.Keyword, 'classdef'), + (whitespace, ' '), + (Token.Name, 'Name'), + (whitespace, ' '), + (Token.Operator, '<'), + (whitespace, ' '), + (Token.Keyword, 'dynamicprops'), + (whitespace, '\n '), + (Token.Keyword, 'properties'), + (whitespace, '\n '), + (Token.Comment, '% i am a comment'), + (whitespace, '\n '), + (Token.Name, 'name1'), + (whitespace, '\n '), + (Token.Name, 'name2'), + (whitespace, '\n '), + (Token.Keyword, 'end'), + (whitespace, '\n '), + (Token.Keyword, 'properties'), + (whitespace, ' '), + (Token.Punctuation, '('), + (Token.Name.Builtin, 'Constant'), + (whitespace, ' '), + (Token.Punctuation, '='), + (whitespace, ' '), + (Token.Keyword, 'true'), + (Token.Punctuation, ','), + (whitespace, ' '), + (Token.Name.Builtin, 'SetAccess'), + (whitespace, ' '), + (Token.Punctuation, '='), + (whitespace, ' '), + (Token.Keyword, 'protected'), + (Token.Punctuation, ')'), + (whitespace, "\n "), + (Token.Comment, '% i too am a comment'), + (whitespace, '\n '), + (Token.Name, 'matrix'), + (whitespace, ' '), + (Token.Punctuation, '='), + (whitespace, ' '), + (Token.Punctuation, '['), + (Token.Literal.Number.Integer, '0'), + (Token.Punctuation, ','), + (whitespace, ' '), + (Token.Literal.Number.Integer, '1'), + (Token.Punctuation, ','), + (whitespace, ' '), + (Token.Literal.Number.Integer, '2'), + (Token.Punctuation, ']'), + (Token.Punctuation, ';'), + (whitespace, '\n '), + (Token.Name, 'string'), + (whitespace, ' '), + (Token.Punctuation, '='), + (whitespace, ' '), + (Token.Literal.String, "'"), + (Token.Literal.String, "i am a string'"), + (whitespace, '\n '), + (Token.Keyword, 'end'), + (whitespace, '\n '), + (Token.Keyword, 'methods'), + (whitespace, '\n '), + (Token.Comment, '% i am also a comment'), + (whitespace, '\n '), + (Token.Keyword, 'function'), + (whitespace, ' '), + (Token.Text, 'self'), + (whitespace, ' '), + (Token.Punctuation, '='), + (whitespace, ' '), + (Token.Name.Function, 'Name'), + (Token.Punctuation, '('), + (Token.Punctuation, ')'), + (whitespace, '\n '), + (Token.Comment, '% i am a comment inside a constructor'), + (whitespace, '\n '), + (Token.Keyword, 'end'), + (whitespace, '\n '), + (Token.Keyword, 'end'), + (whitespace, '\n'), + (Token.Keyword, 'end'), + (whitespace, '\n'), + ] + assert list(lexer.get_tokens(MATLAB_SAMPLE_CLASS)) == tokens + -- cgit v1.2.1