From 14ffe7bc486805c568403757d8634211f8458451 Mon Sep 17 00:00:00 2001 From: Vitaly Cheptsov <4348897+vit9696@users.noreply.github.com> Date: Mon, 4 Oct 2021 16:20:47 +0300 Subject: Implement _Alignas and _Alignof support with tests (#435) * Implement _Alignas and _Alignof support with tests * Improve testing and avoid unnecessary alignas for typedef * Add more tests * Drop legacy artifact * Remove extra _add_declaration_specifier call * Drop custom equality comparators for now Co-authored-by: vit9696 --- examples/func_defs_add_param.py | 1 + pycparser/_ast_gen.py | 1 - pycparser/_c_ast.cfg | 8 +++-- pycparser/c_ast.py | 38 +++++++++++++++++------ pycparser/c_generator.py | 4 +++ pycparser/c_lexer.py | 3 +- pycparser/c_parser.py | 53 +++++++++++++++++++++++++++------ tests/c_files/c11.c | 6 ++++ tests/test_c_generator.py | 21 +++++++++++++ tests/test_c_lexer.py | 1 + tests/test_c_parser.py | 34 +++++++++++++++++++-- utils/fake_libc_include/_fake_defines.h | 6 ++++ utils/fake_libc_include/stdalign.h | 2 ++ 13 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 utils/fake_libc_include/stdalign.h diff --git a/examples/func_defs_add_param.py b/examples/func_defs_add_param.py index e020745..d79fef6 100644 --- a/examples/func_defs_add_param.py +++ b/examples/func_defs_add_param.py @@ -29,6 +29,7 @@ class ParamAdder(c_ast.NodeVisitor): newdecl = c_ast.Decl( name='_hidden', quals=[], + align=[], storage=[], funcspec=[], type=ty, diff --git a/pycparser/_ast_gen.py b/pycparser/_ast_gen.py index ba03d90..027efe7 100644 --- a/pycparser/_ast_gen.py +++ b/pycparser/_ast_gen.py @@ -86,7 +86,6 @@ class NodeCfg(object): src = self._gen_init() src += '\n' + self._gen_children() src += '\n' + self._gen_iter() - src += '\n' + self._gen_attr_names() return src diff --git a/pycparser/_c_ast.cfg b/pycparser/_c_ast.cfg index e9b5685..0626533 100644 --- a/pycparser/_c_ast.cfg +++ b/pycparser/_c_ast.cfg @@ -25,6 +25,8 @@ ArrayRef: [name*, subscript*] # Assignment: [op, lvalue*, rvalue*] +Alignas: [alignment*] + BinaryOp: [op, left*, right*] Break: [] @@ -59,7 +61,7 @@ Continue: [] # init: initialization value, or None # bitsize: bit field size, or None # -Decl: [name, quals, storage, funcspec, type*, init*, bitsize*] +Decl: [name, quals, align, storage, funcspec, type*, init*, bitsize*] DeclList: [decls**] @@ -172,14 +174,14 @@ TernaryOp: [cond*, iftrue*, iffalse*] # A base type declaration # -TypeDecl: [declname, quals, type*] +TypeDecl: [declname, quals, align, type*] # A typedef declaration. # Very similar to Decl, but without some attributes # Typedef: [name, quals, storage, type*] -Typename: [name, quals, type*] +Typename: [name, quals, align, type*] UnaryOp: [op, expr*] diff --git a/pycparser/c_ast.py b/pycparser/c_ast.py index 192c106..6575a2a 100644 --- a/pycparser/c_ast.py +++ b/pycparser/c_ast.py @@ -229,6 +229,23 @@ class Assignment(Node): attr_names = ('op', ) +class Alignas(Node): + __slots__ = ('alignment', 'coord', '__weakref__') + def __init__(self, alignment, coord=None): + self.alignment = alignment + self.coord = coord + + def children(self): + nodelist = [] + if self.alignment is not None: nodelist.append(("alignment", self.alignment)) + return tuple(nodelist) + + def __iter__(self): + if self.alignment is not None: + yield self.alignment + + attr_names = () + class BinaryOp(Node): __slots__ = ('op', 'left', 'right', 'coord', '__weakref__') def __init__(self, op, left, right, coord=None): @@ -379,10 +396,11 @@ class Continue(Node): attr_names = () class Decl(Node): - __slots__ = ('name', 'quals', 'storage', 'funcspec', 'type', 'init', 'bitsize', 'coord', '__weakref__') - def __init__(self, name, quals, storage, funcspec, type, init, bitsize, coord=None): + __slots__ = ('name', 'quals', 'align', 'storage', 'funcspec', 'type', 'init', 'bitsize', 'coord', '__weakref__') + def __init__(self, name, quals, align, storage, funcspec, type, init, bitsize, coord=None): self.name = name self.quals = quals + self.align = align self.storage = storage self.funcspec = funcspec self.type = type @@ -405,7 +423,7 @@ class Decl(Node): if self.bitsize is not None: yield self.bitsize - attr_names = ('name', 'quals', 'storage', 'funcspec', ) + attr_names = ('name', 'quals', 'align', 'storage', 'funcspec', ) class DeclList(Node): __slots__ = ('decls', 'coord', '__weakref__') @@ -972,10 +990,11 @@ class TernaryOp(Node): attr_names = () class TypeDecl(Node): - __slots__ = ('declname', 'quals', 'type', 'coord', '__weakref__') - def __init__(self, declname, quals, type, coord=None): + __slots__ = ('declname', 'quals', 'align', 'type', 'coord', '__weakref__') + def __init__(self, declname, quals, align, type, coord=None): self.declname = declname self.quals = quals + self.align = align self.type = type self.coord = coord @@ -988,7 +1007,7 @@ class TypeDecl(Node): if self.type is not None: yield self.type - attr_names = ('declname', 'quals', ) + attr_names = ('declname', 'quals', 'align', ) class Typedef(Node): __slots__ = ('name', 'quals', 'storage', 'type', 'coord', '__weakref__') @@ -1011,10 +1030,11 @@ class Typedef(Node): attr_names = ('name', 'quals', 'storage', ) class Typename(Node): - __slots__ = ('name', 'quals', 'type', 'coord', '__weakref__') - def __init__(self, name, quals, type, coord=None): + __slots__ = ('name', 'quals', 'align', 'type', 'coord', '__weakref__') + def __init__(self, name, quals, align, type, coord=None): self.name = name self.quals = quals + self.align = align self.type = type self.coord = coord @@ -1027,7 +1047,7 @@ class Typename(Node): if self.type is not None: yield self.type - attr_names = ('name', 'quals', ) + attr_names = ('name', 'quals', 'align', ) class UnaryOp(Node): __slots__ = ('op', 'expr', 'coord', '__weakref__') diff --git a/pycparser/c_generator.py b/pycparser/c_generator.py index ded8c65..fdfe589 100644 --- a/pycparser/c_generator.py +++ b/pycparser/c_generator.py @@ -180,6 +180,9 @@ class CGenerator(object): def visit_Enum(self, n): return self._generate_struct_union_enum(n, name='enum') + def visit_Alignas(self, n): + return '_Alignas({})'.format(self.visit(n.alignment)) + def visit_Enumerator(self, n): if not n.value: return '{indent}{name},\n'.format( @@ -418,6 +421,7 @@ class CGenerator(object): s = '' if n.funcspec: s = ' '.join(n.funcspec) + ' ' if n.storage: s += ' '.join(n.storage) + ' ' + if n.align: s += self.visit(n.align[0]) + ' ' s += self._generate_type(n.type) return s diff --git a/pycparser/c_lexer.py b/pycparser/c_lexer.py index e7f0cb5..8fdd3d7 100644 --- a/pycparser/c_lexer.py +++ b/pycparser/c_lexer.py @@ -111,7 +111,8 @@ class CLexer(object): keywords_new = ( '_BOOL', '_COMPLEX', - '_NORETURN', '_THREAD_LOCAL', '_STATIC_ASSERT', '_ATOMIC', + '_NORETURN', '_THREAD_LOCAL', '_STATIC_ASSERT', + '_ATOMIC', '_ALIGNOF', '_ALIGNAS', ) keyword_map = {} diff --git a/pycparser/c_parser.py b/pycparser/c_parser.py index 8fd8f16..4bbeeca 100644 --- a/pycparser/c_parser.py +++ b/pycparser/c_parser.py @@ -349,6 +349,7 @@ class CParser(PLYParser): * storage: a list of storage type qualifiers * type: a list of type specifiers * function: a list of function specifiers + * alignment: a list of alignment specifiers This method is given a declaration specifier, and a new specifier of a given kind. @@ -357,7 +358,7 @@ class CParser(PLYParser): Returns the declaration specifier, with the new specifier incorporated. """ - spec = declspec or dict(qual=[], storage=[], type=[], function=[]) + spec = declspec or dict(qual=[], storage=[], type=[], function=[], alignment=[]) if append: spec[kind].append(newspec) @@ -398,6 +399,7 @@ class CParser(PLYParser): declname=spec['type'][-1].names[0], type=None, quals=None, + align=spec['alignment'], coord=spec['type'][-1].coord) # Remove the "new" type's name from the end of spec['type'] del spec['type'][-1] @@ -426,6 +428,7 @@ class CParser(PLYParser): declaration = c_ast.Decl( name=None, quals=spec['qual'], + align=spec['alignment'], storage=spec['storage'], funcspec=spec['function'], type=decl['decl'], @@ -583,6 +586,7 @@ class CParser(PLYParser): # no declaration specifiers - 'int' becomes the default type spec = dict( qual=[], + alignment=[], storage=[], type=[c_ast.IdentifierType(['int'], coord=self._token_coord(p, 1))], @@ -703,6 +707,7 @@ class CParser(PLYParser): decls = [c_ast.Decl( name=None, quals=spec['qual'], + align=spec['alignment'], storage=spec['storage'], funcspec=spec['function'], type=ty[0], @@ -786,6 +791,11 @@ class CParser(PLYParser): """ p[0] = self._add_declaration_specifier(p[2], p[1], 'type') + def p_declaration_specifiers_no_type_5(self, p): + """ declaration_specifiers_no_type : alignment_specifier declaration_specifiers_no_type_opt + """ + p[0] = self._add_declaration_specifier(p[2], p[1], 'alignment') + def p_declaration_specifiers_1(self, p): """ declaration_specifiers : declaration_specifiers type_qualifier """ @@ -816,6 +826,11 @@ class CParser(PLYParser): """ p[0] = self._add_declaration_specifier(p[1], p[2], 'type', append=True) + def p_declaration_specifiers_7(self, p): + """ declaration_specifiers : declaration_specifiers alignment_specifier + """ + p[0] = self._add_declaration_specifier(p[1], p[2], 'alignment', append=True) + def p_storage_class_specifier(self, p): """ storage_class_specifier : AUTO | REGISTER @@ -920,7 +935,17 @@ class CParser(PLYParser): def p_specifier_qualifier_list_4(self, p): """ specifier_qualifier_list : type_qualifier_list type_specifier """ - p[0] = dict(qual=p[1], storage=[], type=[p[2]], function=[]) + p[0] = dict(qual=p[1], alignment=[], storage=[], type=[p[2]], function=[]) + + def p_specifier_qualifier_list_5(self, p): + """ specifier_qualifier_list : alignment_specifier + """ + p[0] = dict(qual=[], alignment=[p[1]], storage=[], type=[], function=[]) + + def p_specifier_qualifier_list_6(self, p): + """ specifier_qualifier_list : specifier_qualifier_list alignment_specifier + """ + p[0] = self._add_declaration_specifier(p[1], p[2], 'alignment') # TYPEID is allowed here (and in other struct/enum related tag names), because # struct/enum tags reside in their own namespace and can be named the same as types @@ -1059,7 +1084,7 @@ class CParser(PLYParser): if len(p) > 3: p[0] = {'decl': p[1], 'bitsize': p[3]} else: - p[0] = {'decl': c_ast.TypeDecl(None, None, None), 'bitsize': p[2]} + p[0] = {'decl': c_ast.TypeDecl(None, None, None, None), 'bitsize': p[2]} def p_enum_specifier_1(self, p): """ enum_specifier : ENUM ID @@ -1091,6 +1116,12 @@ class CParser(PLYParser): p[1].enumerators.append(p[3]) p[0] = p[1] + def p_alignment_specifier(self, p): + """ alignment_specifier : _ALIGNAS LPAREN type_name RPAREN + | _ALIGNAS LPAREN constant_expression RPAREN + """ + p[0] = c_ast.Alignas(p[3], self._token_coord(p, 1)) + def p_enumerator(self, p): """ enumerator : ID | ID EQUALS constant_expression @@ -1133,6 +1164,7 @@ class CParser(PLYParser): declname=p[1], type=None, quals=None, + align=None, coord=self._token_coord(p, 1)) @parameterized(('id', 'ID'), ('typeid', 'TYPEID')) @@ -1320,7 +1352,8 @@ class CParser(PLYParser): decl = c_ast.Typename( name='', quals=spec['qual'], - type=p[2] or c_ast.TypeDecl(None, None, None), + align=None, + type=p[2] or c_ast.TypeDecl(None, None, None, None), coord=self._token_coord(p, 2)) typename = spec['type'] decl = self._fix_decl_name_type(decl, typename) @@ -1389,7 +1422,8 @@ class CParser(PLYParser): typename = c_ast.Typename( name='', quals=p[1]['qual'][:], - type=p[2] or c_ast.TypeDecl(None, None, None), + align=None, + type=p[2] or c_ast.TypeDecl(None, None, None, None), coord=self._token_coord(p, 2)) p[0] = self._fix_decl_name_type(typename, p[1]['type']) @@ -1397,7 +1431,7 @@ class CParser(PLYParser): def p_abstract_declarator_1(self, p): """ abstract_declarator : pointer """ - dummytype = c_ast.TypeDecl(None, None, None) + dummytype = c_ast.TypeDecl(None, None, None, None) p[0] = self._type_modify_decl( decl=dummytype, modifier=p[1]) @@ -1437,7 +1471,7 @@ class CParser(PLYParser): """ quals = (p[2] if len(p) > 4 else []) or [] p[0] = c_ast.ArrayDecl( - type=c_ast.TypeDecl(None, None, None), + type=c_ast.TypeDecl(None, None, None, None), dim=p[3] if len(p) > 4 else p[2], dim_quals=quals, coord=self._token_coord(p, 1)) @@ -1457,7 +1491,7 @@ class CParser(PLYParser): """ direct_abstract_declarator : LBRACKET TIMES RBRACKET """ p[0] = c_ast.ArrayDecl( - type=c_ast.TypeDecl(None, None, None), + type=c_ast.TypeDecl(None, None, None, None), dim=c_ast.ID(p[3], self._token_coord(p, 3)), dim_quals=[], coord=self._token_coord(p, 1)) @@ -1477,7 +1511,7 @@ class CParser(PLYParser): """ p[0] = c_ast.FuncDecl( args=p[2], - type=c_ast.TypeDecl(None, None, None), + type=c_ast.TypeDecl(None, None, None, None), coord=self._token_coord(p, 1)) # declaration is a list, statement isn't. To make it consistent, block_item @@ -1682,6 +1716,7 @@ class CParser(PLYParser): def p_unary_expression_3(self, p): """ unary_expression : SIZEOF unary_expression | SIZEOF LPAREN type_name RPAREN + | _ALIGNOF LPAREN type_name RPAREN """ p[0] = c_ast.UnaryOp( p[1], diff --git a/tests/c_files/c11.c b/tests/c_files/c11.c index 3c57f55..4f97a87 100644 --- a/tests/c_files/c11.c +++ b/tests/c_files/c11.c @@ -4,6 +4,7 @@ #include #include #include +#include /* C11 thread locals */ _Thread_local int flag; @@ -12,6 +13,9 @@ _Atomic int flag3; _Atomic(int) flag4; _Atomic(_Atomic(int) *) flag5; atomic_bool flag6; +_Alignas(32) int q32; +_Alignas(long long) int qll; +alignas(64) int qqq; static_assert(sizeof(flag) == sizeof(flag2), "Really unexpected size difference"); @@ -31,6 +35,8 @@ int main() static_assert(sizeof(flag) == sizeof(flag2), "Unexpected size difference"); static_assert(sizeof(flag) == sizeof(flag3), "Unexpected size difference"); static_assert(sizeof(flag) == sizeof(flag4), "Unexpected size difference"); + static_assert(_Alignof(int) == sizeof(int), "Unexpected int alignment"); + static_assert(alignof(int) == sizeof(int), "Unexpected int alignment"); printf("Flag: %d\n", flag); printf("Flag2: %d\n", flag2); diff --git a/tests/test_c_generator.py b/tests/test_c_generator.py index fb9499a..2095e63 100644 --- a/tests/test_c_generator.py +++ b/tests/test_c_generator.py @@ -96,6 +96,27 @@ class TestCtoC(unittest.TestCase): self._assert_ctoc_correct('int test(const char* const* arg);') self._assert_ctoc_correct('int test(const char** const arg);') + # FIXME: These require custom equality comparison. + # def test_alignment(self): + # self._assert_ctoc_correct('_Alignas(32) int b;') + # self._assert_ctoc_correct('int _Alignas(32) a;') + # self._assert_ctoc_correct('_Alignas(32) _Atomic(int) b;') + # self._assert_ctoc_correct('_Atomic(int) _Alignas(32) b;') + # self._assert_ctoc_correct('_Alignas(long long) int a;') + # self._assert_ctoc_correct('int _Alignas(long long) a;') + # self._assert_ctoc_correct(r''' + # typedef struct node_t { + # _Alignas(64) void* next; + # int data; + # } node; + # ''') + # self._assert_ctoc_correct(r''' + # typedef struct node_t { + # void _Alignas(64) * next; + # int data; + # } node; + # ''') + def test_ternary(self): self._assert_ctoc_correct(''' int main(void) diff --git a/tests/test_c_lexer.py b/tests/test_c_lexer.py index e446434..1d3c39b 100644 --- a/tests/test_c_lexer.py +++ b/tests/test_c_lexer.py @@ -94,6 +94,7 @@ class TestCLexerNoErrors(unittest.TestCase): def test_new_keywords(self): self.assertTokensTypes('_Bool', ['_BOOL']) self.assertTokensTypes('_Atomic', ['_ATOMIC']) + self.assertTokensTypes('_Alignas _Alignof', ['_ALIGNAS', '_ALIGNOF']) def test_floating_constants(self): self.assertTokensTypes('1.5f', ['FLOAT_CONST']) diff --git a/tests/test_c_parser.py b/tests/test_c_parser.py index 7e6a648..5b9d916 100755 --- a/tests/test_c_parser.py +++ b/tests/test_c_parser.py @@ -40,16 +40,21 @@ def expand_decl(decl): assert isinstance(decl.values, EnumeratorList) values = [enum.name for enum in decl.values.enumerators] return ['Enum', decl.name, values] + elif typ == Alignas: + return ['Alignas', expand_init(decl.alignment)] elif typ == StaticAssert: return ['StaticAssert', decl.cond.value, decl.message.value] else: nested = expand_decl(decl.type) if typ == Decl: + r = ['Decl'] if decl.quals: - return ['Decl', decl.quals, decl.name, nested] - else: - return ['Decl', decl.name, nested] + r.append(decl.quals) + if decl.align: + r.append(expand_decl(decl.align[0])) + r.extend([decl.name, nested]) + return r elif typ == Typename: # for function parameters if decl.quals: return ['Typename', decl.quals, nested] @@ -98,6 +103,8 @@ def expand_init(init): if init.block_items: blocks = [expand_init(i) for i in init.block_items] return ['Compound', blocks] + elif typ == Typename: + return expand_decl(init) else: # Fallback to type name return [typ.__name__] @@ -618,6 +625,27 @@ class TestCParser_fundamentals(TestCParser_base): ['TypeDecl', ['IdentifierType', ['int']]]]]]) + def test_alignof(self): + r = self.parse('int a = _Alignof(int);') + self.assertEqual(expand_decl(r.ext[0]), ['Decl', 'a', ['TypeDecl', ['IdentifierType', ['int']]]]) + self.assertEqual(expand_init(r.ext[0].init), ['UnaryOp', '_Alignof', + ['Typename', ['TypeDecl', ['IdentifierType', ['int']]]]]) + + self.assertEqual(expand_decl(self.parse('_Alignas(_Alignof(int)) char a;').ext[0]), + ['Decl', ['Alignas', + ['UnaryOp', '_Alignof', ['Typename', ['TypeDecl', ['IdentifierType', ['int']]]]]], + 'a', ['TypeDecl', ['IdentifierType', ['char']]]]) + + self.assertEqual(expand_decl(self.parse('_Alignas(4) char a;').ext[0]), + ['Decl', ['Alignas', + ['Constant', 'int', '4']], + 'a', ['TypeDecl', ['IdentifierType', ['char']]]]) + + self.assertEqual(expand_decl(self.parse('_Alignas(int) char a;').ext[0]), + ['Decl', ['Alignas', + ['Typename', ['TypeDecl', ['IdentifierType', ['int']]]]], + 'a', ['TypeDecl', ['IdentifierType', ['char']]]]) + def test_offsetof(self): def expand_ref(n): if isinstance(n, StructRef): diff --git a/utils/fake_libc_include/_fake_defines.h b/utils/fake_libc_include/_fake_defines.h index 86b5c66..678cffc 100644 --- a/utils/fake_libc_include/_fake_defines.h +++ b/utils/fake_libc_include/_fake_defines.h @@ -251,4 +251,10 @@ #define ATOMIC_FLAG_INIT { 0 } #define kill_dependency(y) (y) +/* C11 stdalign.h defines */ +#define alignas _Alignas +#define alignof _Alignof +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 + #endif diff --git a/utils/fake_libc_include/stdalign.h b/utils/fake_libc_include/stdalign.h new file mode 100644 index 0000000..f952c1d --- /dev/null +++ b/utils/fake_libc_include/stdalign.h @@ -0,0 +1,2 @@ +#include "_fake_defines.h" +#include "_fake_typedefs.h" -- cgit v1.2.1