diff options
Diffstat (limited to 'Tools/c-analyzer/c_parser/info.py')
| -rw-r--r-- | Tools/c-analyzer/c_parser/info.py | 1658 |
1 files changed, 1658 insertions, 0 deletions
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py new file mode 100644 index 0000000000..a07ce2e0cc --- /dev/null +++ b/Tools/c-analyzer/c_parser/info.py @@ -0,0 +1,1658 @@ +from collections import namedtuple +import enum +import os.path +import re + +from c_common.clsutil import classonly +import c_common.misc as _misc +import c_common.strutil as _strutil +import c_common.tables as _tables +from .parser._regexes import SIMPLE_TYPE + + +FIXED_TYPE = _misc.Labeled('FIXED_TYPE') + +POTS_REGEX = re.compile(rf'^{SIMPLE_TYPE}$', re.VERBOSE) + + +def is_pots(typespec): + if not typespec: + return None + if type(typespec) is not str: + _, _, _, typespec, _ = get_parsed_vartype(typespec) + return POTS_REGEX.match(typespec) is not None + + +def is_funcptr(vartype): + if not vartype: + return None + _, _, _, _, abstract = get_parsed_vartype(vartype) + return _is_funcptr(abstract) + + +def _is_funcptr(declstr): + if not declstr: + return None + # XXX Support "(<name>*)(". + return '(*)(' in declstr.replace(' ', '') + + +def is_exported_symbol(decl): + _, storage, _, _, _ = get_parsed_vartype(decl) + raise NotImplementedError + + +def is_process_global(vardecl): + kind, storage, _, _, _ = get_parsed_vartype(vardecl) + if kind is not KIND.VARIABLE: + raise NotImplementedError(vardecl) + if 'static' in (storage or ''): + return True + + if hasattr(vardecl, 'parent'): + parent = vardecl.parent + else: + parent = vardecl.get('parent') + return not parent + + +def is_fixed_type(vardecl): + if not vardecl: + return None + _, _, _, typespec, abstract = get_parsed_vartype(vardecl) + if 'typeof' in typespec: + raise NotImplementedError(vardecl) + elif not abstract: + return True + + if '*' not in abstract: + # XXX What about []? + return True + elif _is_funcptr(abstract): + return True + else: + for after in abstract.split('*')[1:]: + if not after.lstrip().startswith('const'): + return False + else: + return True + + +def is_immutable(vardecl): + if not vardecl: + return None + if not is_fixed_type(vardecl): + return False + _, _, typequal, _, _ = get_parsed_vartype(vardecl) + # If there, it can only be "const" or "volatile". + return typequal == 'const' + + +############################# +# kinds + +@enum.unique +class KIND(enum.Enum): + + # XXX Use these in the raw parser code. + TYPEDEF = 'typedef' + STRUCT = 'struct' + UNION = 'union' + ENUM = 'enum' + FUNCTION = 'function' + VARIABLE = 'variable' + STATEMENT = 'statement' + + @classonly + def _from_raw(cls, raw): + if raw is None: + return None + elif isinstance(raw, cls): + return raw + elif type(raw) is str: + # We could use cls[raw] for the upper-case form, + # but there's no need to go to the trouble. + return cls(raw.lower()) + else: + raise NotImplementedError(raw) + + @classonly + def by_priority(cls, group=None): + if group is None: + return cls._ALL_BY_PRIORITY.copy() + elif group == 'type': + return cls._TYPE_DECLS_BY_PRIORITY.copy() + elif group == 'decl': + return cls._ALL_DECLS_BY_PRIORITY.copy() + elif isinstance(group, str): + raise NotImplementedError(group) + else: + # XXX Treat group as a set of kinds & return in priority order? + raise NotImplementedError(group) + + @classonly + def is_type_decl(cls, kind): + if kind in cls.TYPES: + return True + if not isinstance(kind, cls): + raise TypeError(f'expected KIND, got {kind!r}') + return False + + @classonly + def is_decl(cls, kind): + if kind in cls.DECLS: + return True + if not isinstance(kind, cls): + raise TypeError(f'expected KIND, got {kind!r}') + return False + + @classonly + def get_group(cls, kind, *, groups=None): + if not isinstance(kind, cls): + raise TypeError(f'expected KIND, got {kind!r}') + if groups is None: + groups = ['type'] + elif not groups: + groups = () + elif isinstance(groups, str): + group = groups + if group not in cls._GROUPS: + raise ValueError(f'unsupported group {group!r}') + groups = [group] + else: + unsupported = [g for g in groups if g not in cls._GROUPS] + if unsupported: + raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}') + for group in groups: + if kind in cls._GROUPS[group]: + return group + else: + return kind.value + + @classonly + def resolve_group(cls, group): + if isinstance(group, cls): + return {group} + elif isinstance(group, str): + try: + return cls._GROUPS[group].copy() + except KeyError: + raise ValueError(f'unsupported group {group!r}') + else: + resolved = set() + for gr in group: + resolve.update(cls.resolve_group(gr)) + return resolved + #return {*cls.resolve_group(g) for g in group} + + +KIND._TYPE_DECLS_BY_PRIORITY = [ + # These are in preferred order. + KIND.TYPEDEF, + KIND.STRUCT, + KIND.UNION, + KIND.ENUM, +] +KIND._ALL_DECLS_BY_PRIORITY = [ + # These are in preferred order. + *KIND._TYPE_DECLS_BY_PRIORITY, + KIND.FUNCTION, + KIND.VARIABLE, +] +KIND._ALL_BY_PRIORITY = [ + # These are in preferred order. + *KIND._ALL_DECLS_BY_PRIORITY, + KIND.STATEMENT, +] + +KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY) +KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY) +KIND._GROUPS = { + 'type': KIND.TYPES, + 'decl': KIND.DECLS, +} +KIND._GROUPS.update((k.value, {k}) for k in KIND) + + +# The module-level kind-related helpers (below) deal with <item>.kind: + +def is_type_decl(kind): + # Handle ParsedItem, Declaration, etc.. + kind = getattr(kind, 'kind', kind) + return KIND.is_type_decl(kind) + + +def is_decl(kind): + # Handle ParsedItem, Declaration, etc.. + kind = getattr(kind, 'kind', kind) + return KIND.is_decl(kind) + + +def filter_by_kind(items, kind): + if kind == 'type': + kinds = KIND._TYPE_DECLS + elif kind == 'decl': + kinds = KIND._TYPE_DECLS + try: + okay = kind in KIND + except TypeError: + kinds = set(kind) + else: + kinds = {kind} if okay else set(kind) + for item in items: + if item.kind in kinds: + yield item + + +def collate_by_kind(items): + collated = {kind: [] for kind in KIND} + for item in items: + try: + collated[item.kind].append(item) + except KeyError: + raise ValueError(f'unsupported kind in {item!r}') + return collated + + +def get_kind_group(kind): + # Handle ParsedItem, Declaration, etc.. + kind = getattr(kind, 'kind', kind) + return KIND.get_group(kind) + + +def collate_by_kind_group(items): + collated = {KIND.get_group(k): [] for k in KIND} + for item in items: + group = KIND.get_group(item.kind) + collated[group].append(item) + return collated + + +############################# +# low-level + +class FileInfo(namedtuple('FileInfo', 'filename lno')): + @classmethod + def from_raw(cls, raw): + if isinstance(raw, cls): + return raw + elif isinstance(raw, tuple): + return cls(*raw) + elif not raw: + return None + elif isinstance(raw, str): + return cls(raw, -1) + else: + raise TypeError(f'unsupported "raw": {raw:!r}') + + def __str__(self): + return self.filename + + def fix_filename(self, relroot): + filename = os.path.relpath(self.filename, relroot) + return self._replace(filename=filename) + + +class SourceLine(namedtuple('Line', 'file kind data conditions')): + KINDS = ( + #'directive', # data is ... + 'source', # "data" is the line + #'comment', # "data" is the text, including comment markers + ) + + @property + def filename(self): + return self.file.filename + + @property + def lno(self): + return self.file.lno + + +class DeclID(namedtuple('DeclID', 'filename funcname name')): + """The globally-unique identifier for a declaration.""" + + @classmethod + def from_row(cls, row, **markers): + row = _tables.fix_row(row, **markers) + return cls(*row) + + def __new__(cls, filename, funcname, name): + self = super().__new__( + cls, + filename=str(filename) if filename else None, + funcname=str(funcname) if funcname else None, + name=str(name) if name else None, + ) + self._compare = tuple(v or '' for v in self) + return self + + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + try: + other = tuple(v or '' for v in other) + except TypeError: + return NotImplemented + return self._compare == other + + def __gt__(self, other): + try: + other = tuple(v or '' for v in other) + except TypeError: + return NotImplemented + return self._compare > other + + +class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')): + + @classmethod + def from_raw(cls, raw): + if isinstance(raw, cls): + return raw + elif isinstance(raw, tuple): + return cls(*raw) + else: + raise TypeError(f'unsupported "raw": {raw:!r}') + + @classmethod + def from_row(cls, row, columns=None): + if not columns: + colnames = 'filename funcname name kind data'.split() + else: + colnames = list(columns) + for i, column in enumerate(colnames): + if column == 'file': + colnames[i] = 'filename' + elif column == 'funcname': + colnames[i] = 'parent' + if len(row) != len(set(colnames)): + raise NotImplementedError(columns, row) + kwargs = {} + for column, value in zip(colnames, row): + if column == 'filename': + kwargs['file'] = FileInfo.from_raw(value) + elif column == 'kind': + kwargs['kind'] = KIND(value) + elif column in cls._fields: + kwargs[column] = value + else: + raise NotImplementedError(column) + return cls(**kwargs) + + @property + def id(self): + try: + return self._id + except AttributeError: + if self.kind is KIND.STATEMENT: + self._id = None + else: + self._id = DeclID(str(self.file), self.funcname, self.name) + return self._id + + @property + def filename(self): + if not self.file: + return None + return self.file.filename + + @property + def lno(self): + if not self.file: + return -1 + return self.file.lno + + @property + def funcname(self): + if not self.parent: + return None + if type(self.parent) is str: + return self.parent + else: + return self.parent.name + + def as_row(self, columns=None): + if not columns: + columns = self._fields + row = [] + for column in columns: + if column == 'file': + value = self.filename + elif column == 'kind': + value = self.kind.value + elif column == 'data': + value = self._render_data() + else: + value = getattr(self, column) + row.append(value) + return row + + def _render_data(self): + if not self.data: + return None + elif isinstance(self.data, str): + return self.data + else: + # XXX + raise NotImplementedError + + +def _get_vartype(data): + try: + vartype = dict(data['vartype']) + except KeyError: + vartype = dict(data) + storage = data.get('storage') + else: + storage = data.get('storage') or vartype.get('storage') + del vartype['storage'] + return storage, vartype + + +def get_parsed_vartype(decl): + kind = getattr(decl, 'kind', None) + if isinstance(decl, ParsedItem): + storage, vartype = _get_vartype(decl.data) + typequal = vartype['typequal'] + typespec = vartype['typespec'] + abstract = vartype['abstract'] + elif isinstance(decl, dict): + kind = decl.get('kind') + storage, vartype = _get_vartype(decl) + typequal = vartype['typequal'] + typespec = vartype['typespec'] + abstract = vartype['abstract'] + elif isinstance(decl, VarType): + storage = None + typequal, typespec, abstract = decl + elif isinstance(decl, TypeDef): + storage = None + typequal, typespec, abstract = decl.vartype + elif isinstance(decl, Variable): + storage = decl.storage + typequal, typespec, abstract = decl.vartype + elif isinstance(decl, Function): + storage = decl.storage + typequal, typespec, abstract = decl.signature.returntype + elif isinstance(decl, str): + vartype, storage = VarType.from_str(decl) + typequal, typespec, abstract = vartype + else: + raise NotImplementedError(decl) + return kind, storage, typequal, typespec, abstract + + +############################# +# high-level + +class HighlevelParsedItem: + + kind = None + + FIELDS = ('file', 'parent', 'name', 'data') + + @classmethod + def from_parsed(cls, parsed): + if parsed.kind is not cls.kind: + raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})') + data, extra = cls._resolve_data(parsed.data) + self = cls( + cls._resolve_file(parsed), + parsed.name, + data, + cls._resolve_parent(parsed) if parsed.parent else None, + **extra or {} + ) + self._parsed = parsed + return self + + @classmethod + def _resolve_file(cls, parsed): + fileinfo = FileInfo.from_raw(parsed.file) + if not fileinfo: + raise NotImplementedError(parsed) + return fileinfo + + @classmethod + def _resolve_data(cls, data): + return data, None + + @classmethod + def _raw_data(cls, data, extra): + if isinstance(data, str): + return data + else: + raise NotImplementedError(data) + + @classmethod + def _data_as_row(cls, data, extra, colnames): + row = {} + for colname in colnames: + if colname in row: + continue + rendered = cls._render_data_row_item(colname, data, extra) + if rendered is iter(rendered): + rendered, = rendered + row[colname] = rendered + return row + + @classmethod + def _render_data_row_item(cls, colname, data, extra): + if colname == 'data': + return str(data) + else: + return None + + @classmethod + def _render_data_row(cls, fmt, data, extra, colnames): + if fmt != 'row': + raise NotImplementedError + datarow = cls._data_as_row(data, extra, colnames) + unresolved = [c for c, v in datarow.items() if v is None] + if unresolved: + raise NotImplementedError(unresolved) + for colname, value in datarow.items(): + if type(value) != str: + if colname == 'kind': + datarow[colname] = value.value + else: + datarow[colname] = str(value) + return datarow + + @classmethod + def _render_data(cls, fmt, data, extra): + row = cls._render_data_row(fmt, data, extra, ['data']) + yield ' '.join(row.values()) + + @classmethod + def _resolve_parent(cls, parsed, *, _kind=None): + fileinfo = FileInfo(parsed.file.filename, -1) + if isinstance(parsed.parent, str): + if parsed.parent.isidentifier(): + name = parsed.parent + else: + # XXX It could be something like "<kind> <name>". + raise NotImplementedError(repr(parsed.parent)) + parent = ParsedItem(fileinfo, _kind, None, name, None) + elif type(parsed.parent) is tuple: + # XXX It could be something like (kind, name). + raise NotImplementedError(repr(parsed.parent)) + else: + return parsed.parent + Parent = KIND_CLASSES.get(_kind, Declaration) + return Parent.from_parsed(parent) + + @classmethod + def _parse_columns(cls, columns): + colnames = {} # {requested -> actual} + columns = list(columns or cls.FIELDS) + datacolumns = [] + for i, colname in enumerate(columns): + if colname == 'file': + columns[i] = 'filename' + colnames['file'] = 'filename' + elif colname == 'lno': + columns[i] = 'line' + colnames['lno'] = 'line' + elif colname in ('filename', 'line'): + colnames[colname] = colname + elif colname == 'data': + datacolumns.append(colname) + colnames[colname] = None + elif colname in cls.FIELDS or colname == 'kind': + colnames[colname] = colname + else: + datacolumns.append(colname) + colnames[colname] = None + return columns, datacolumns, colnames + + def __init__(self, file, name, data, parent=None, *, + _extra=None, + _shortkey=None, + _key=None, + ): + self.file = file + self.parent = parent or None + self.name = name + self.data = data + self._extra = _extra or {} + self._shortkey = _shortkey + self._key = _key + + def __repr__(self): + args = [f'{n}={getattr(self, n)!r}' + for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]] + return f'{type(self).__name__}({", ".join(args)})' + + def __str__(self): + try: + return self._str + except AttributeError: + self._str = next(self.render()) + return self._str + + def __getattr__(self, name): + try: + return self._extra[name] + except KeyError: + raise AttributeError(name) + + def __hash__(self): + return hash(self._key) + + def __eq__(self, other): + if isinstance(other, HighlevelParsedItem): + return self._key == other._key + elif type(other) is tuple: + return self._key == other + else: + return NotImplemented + + def __gt__(self, other): + if isinstance(other, HighlevelParsedItem): + return self._key > other._key + elif type(other) is tuple: + return self._key > other + else: + return NotImplemented + + @property + def id(self): + return self.parsed.id + + @property + def shortkey(self): + return self._shortkey + + @property + def key(self): + return self._key + + @property + def filename(self): + if not self.file: + return None + return self.file.filename + + @property + def parsed(self): + try: + return self._parsed + except AttributeError: + parent = self.parent + if parent is not None and not isinstance(parent, str): + parent = parent.name + self._parsed = ParsedItem( + self.file, + self.kind, + parent, + self.name, + self._raw_data(), + ) + return self._parsed + + def fix_filename(self, relroot): + if self.file: + self.file = self.file.fix_filename(relroot) + + def as_rowdata(self, columns=None): + columns, datacolumns, colnames = self._parse_columns(columns) + return self._as_row(colnames, datacolumns, self._data_as_row) + + def render_rowdata(self, columns=None): + columns, datacolumns, colnames = self._parse_columns(columns) + def data_as_row(data, ext, cols): + return self._render_data_row('row', data, ext, cols) + rowdata = self._as_row(colnames, datacolumns, data_as_row) + for column, value in rowdata.items(): + colname = colnames.get(column) + if not colname: + continue + if column == 'kind': + value = value.value + else: + if column == 'parent': + if self.parent: + value = f'({self.parent.kind.value} {self.parent.name})' + if not value: + value = '-' + elif type(value) is VarType: + value = repr(str(value)) + else: + value = str(value) + rowdata[column] = value + return rowdata + + def _as_row(self, colnames, datacolumns, data_as_row): + try: + data = data_as_row(self.data, self._extra, datacolumns) + except NotImplementedError: + data = None + row = data or {} + for column, colname in colnames.items(): + if colname == 'filename': + value = self.file.filename if self.file else None + elif colname == 'line': + value = self.file.lno if self.file else None + elif colname is None: + value = getattr(self, column, None) + else: + value = getattr(self, colname, None) + row.setdefault(column, value) + return row + + def render(self, fmt='line'): + fmt = fmt or 'line' + try: + render = _FORMATS[fmt] + except KeyError: + raise TypeError(f'unsupported fmt {fmt!r}') + try: + data = self._render_data(fmt, self.data, self._extra) + except NotImplementedError: + data = '-' + yield from render(self, data) + + +### formats ### + +def _fmt_line(parsed, data=None): + parts = [ + f'<{parsed.kind.value}>', + ] + parent = '' + if parsed.parent: + parent = parsed.parent + if not isinstance(parent, str): + if parent.kind is KIND.FUNCTION: + parent = f'{parent.name}()' + else: + parent = parent.name + name = f'<{parent}>.{parsed.name}' + else: + name = parsed.name + if data is None: + data = parsed.data + elif data is iter(data): + data, = data + parts.extend([ + name, + f'<{data}>' if data else '-', + f'({str(parsed.file or "<unknown file>")})', + ]) + yield '\t'.join(parts) + + +def _fmt_full(parsed, data=None): + if parsed.kind is KIND.VARIABLE and parsed.parent: + prefix = 'local ' + suffix = f' ({parsed.parent.name})' + else: + # XXX Show other prefixes (e.g. global, public) + prefix = suffix = '' + yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}' + for column, info in parsed.render_rowdata().items(): + if column == 'kind': + continue + if column == 'name': + continue + if column == 'parent' and parsed.kind is not KIND.VARIABLE: + continue + if column == 'data': + if parsed.kind in (KIND.STRUCT, KIND.UNION): + column = 'members' + elif parsed.kind is KIND.ENUM: + column = 'enumerators' + elif parsed.kind is KIND.STATEMENT: + column = 'text' + data, = data + else: + column = 'signature' + data, = data + if not data: +# yield f'\t{column}:\t-' + continue + elif isinstance(data, str): + yield f'\t{column}:\t{data!r}' + else: + yield f'\t{column}:' + for line in data: + yield f'\t\t- {line}' + else: + yield f'\t{column}:\t{info}' + + +_FORMATS = { + 'raw': (lambda v, _d: [repr(v)]), + 'brief': _fmt_line, + 'line': _fmt_line, + 'full': _fmt_full, +} + + +### declarations ## + +class Declaration(HighlevelParsedItem): + + @classmethod + def from_row(cls, row, **markers): + fixed = tuple(_tables.fix_row(row, **markers)) + if cls is Declaration: + _, _, _, kind, _ = fixed + sub = KIND_CLASSES.get(KIND(kind)) + if not sub or not issubclass(sub, Declaration): + raise TypeError(f'unsupported kind, got {row!r}') + else: + sub = cls + return sub._from_row(fixed) + + @classmethod + def _from_row(cls, row): + filename, funcname, name, kind, data = row + kind = KIND._from_raw(kind) + if kind is not cls.kind: + raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}') + fileinfo = FileInfo.from_raw(filename) + if isinstance(data, str): + data, extra = cls._parse_data(data, fmt='row') + if extra: + return cls(fileinfo, name, data, funcname, _extra=extra) + else: + return cls(fileinfo, name, data, funcname) + + @classmethod + def _resolve_parent(cls, parsed, *, _kind=None): + if _kind is None: + raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})') + return super()._resolve_parent(parsed, _kind=_kind) + + @classmethod + def _render_data(cls, fmt, data, extra): + if not data: + # XXX There should be some! Forward? + yield '???' + else: + yield from cls._format_data(fmt, data, extra) + + @classmethod + def _render_data_row_item(cls, colname, data, extra): + if colname == 'data': + return cls._format_data('row', data, extra) + else: + return None + + @classmethod + def _format_data(cls, fmt, data, extra): + raise NotImplementedError(fmt) + + @classmethod + def _parse_data(cls, datastr, fmt=None): + """This is the reverse of _render_data.""" + if not datastr or datastr is _tables.UNKNOWN or datastr == '???': + return None, None + elif datastr is _tables.EMPTY or datastr == '-': + # All the kinds have *something* even it is unknown. + raise TypeError('all declarations have data of some sort, got none') + else: + return cls._unformat_data(datastr, fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + raise NotImplementedError(fmt) + + +class VarType(namedtuple('VarType', 'typequal typespec abstract')): + + @classmethod + def from_str(cls, text): + orig = text + storage, sep, text = text.strip().partition(' ') + if not sep: + text = storage + storage = None + elif storage not in ('auto', 'register', 'static', 'extern'): + text = orig + storage = None + return cls._from_str(text), storage + + @classmethod + def _from_str(cls, text): + orig = text + if text.startswith(('const ', 'volatile ')): + typequal, _, text = text.partition(' ') + else: + typequal = None + + # Extract a series of identifiers/keywords. + m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text) + if not m: + raise ValueError(f'invalid vartype text {orig!r}') + typespec, abstract = m.groups() + + return cls(typequal, typespec, abstract or None) + + def __str__(self): + parts = [] + if self.qualifier: + parts.append(self.qualifier) + parts.append(self.spec + (self.abstract or '')) + return ' '.join(parts) + + @property + def qualifier(self): + return self.typequal + + @property + def spec(self): + return self.typespec + + +class Variable(Declaration): + kind = KIND.VARIABLE + + @classmethod + def _resolve_parent(cls, parsed): + return super()._resolve_parent(parsed, _kind=KIND.FUNCTION) + + @classmethod + def _resolve_data(cls, data): + if not data: + return None, None + storage, vartype = _get_vartype(data) + return VarType(**vartype), {'storage': storage} + + @classmethod + def _raw_data(self, data, extra): + vartype = data._asdict() + return { + 'storage': extra['storage'], + 'vartype': vartype, + } + + @classmethod + def _format_data(cls, fmt, data, extra): + storage = extra.get('storage') + text = f'{storage} {data}' if storage else str(data) + if fmt in ('line', 'brief'): + yield text + #elif fmt == 'full': + elif fmt == 'row': + yield text + else: + raise NotImplementedError(fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + if fmt in ('line', 'brief'): + vartype, storage = VarType.from_str(datastr) + return vartype, {'storage': storage} + #elif fmt == 'full': + elif fmt == 'row': + vartype, storage = VarType.from_str(datastr) + return vartype, {'storage': storage} + else: + raise NotImplementedError(fmt) + + def __init__(self, file, name, data, parent=None, storage=None): + super().__init__(file, name, data, parent, + _extra={'storage': storage}, + _shortkey=f'({parent.name}).{name}' if parent else name, + _key=(str(file), + # Tilde comes after all other ascii characters. + f'~{parent or ""}~', + name, + ), + ) + + @property + def vartype(self): + return self.data + + +class Signature(namedtuple('Signature', 'params returntype inline isforward')): + + @classmethod + def from_str(cls, text): + orig = text + storage, sep, text = text.strip().partition(' ') + if not sep: + text = storage + storage = None + elif storage not in ('auto', 'register', 'static', 'extern'): + text = orig + storage = None + return cls._from_str(text), storage + + @classmethod + def _from_str(cls, text): + orig = text + inline, sep, text = text.partition('|') + if not sep: + text = inline + inline = None + + isforward = False + if text.endswith(';'): + text = text[:-1] + isforward = True + elif text.endswith('{}'): + text = text[:-2] + + index = text.rindex('(') + if index < 0: + raise ValueError(f'bad signature text {orig!r}') + params = text[index:] + while params.count('(') <= params.count(')'): + index = text.rindex('(', 0, index) + if index < 0: + raise ValueError(f'bad signature text {orig!r}') + params = text[index:] + text = text[:index] + + returntype = VarType._from_str(text.rstrip()) + + return cls(params, returntype, inline, isforward) + + def __str__(self): + parts = [] + if self.inline: + parts.extend([ + self.inline, + '|', + ]) + parts.extend([ + str(self.returntype), + self.params, + ';' if self.isforward else '{}', + ]) + return ' '.join(parts) + + @property + def returns(self): + return self.returntype + + +class Function(Declaration): + kind = KIND.FUNCTION + + @classmethod + def _resolve_data(cls, data): + if not data: + return None, None + kwargs = dict(data) + returntype = dict(data['returntype']) + del returntype['storage'] + kwargs['returntype'] = VarType(**returntype) + storage = kwargs.pop('storage') + return Signature(**kwargs), {'storage': storage} + + @classmethod + def _raw_data(self, data): + # XXX finsh! + return data + + @classmethod + def _format_data(cls, fmt, data, extra): + storage = extra.get('storage') + text = f'{storage} {data}' if storage else str(data) + if fmt in ('line', 'brief'): + yield text + #elif fmt == 'full': + elif fmt == 'row': + yield text + else: + raise NotImplementedError(fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + if fmt in ('line', 'brief'): + sig, storage = Signature.from_str(sig) + return sig, {'storage': storage} + #elif fmt == 'full': + elif fmt == 'row': + sig, storage = Signature.from_str(sig) + return sig, {'storage': storage} + else: + raise NotImplementedError(fmt) + + def __init__(self, file, name, data, parent=None, storage=None): + super().__init__(file, name, data, parent, _extra={'storage': storage}) + self._shortkey = f'~{name}~ {self.data}' + self._key = ( + str(file), + self._shortkey, + ) + + @property + def signature(self): + return self.data + + +class TypeDeclaration(Declaration): + + def __init__(self, file, name, data, parent=None, *, _shortkey=None): + if not _shortkey: + _shortkey = f'{self.kind.value} {name}' + super().__init__(file, name, data, parent, + _shortkey=_shortkey, + _key=( + str(file), + _shortkey, + ), + ) + + +class POTSType(TypeDeclaration): + + def __init__(self, name): + _file = _data = _parent = None + super().__init__(_file, name, _data, _parent, _shortkey=name) + + +class FuncPtr(TypeDeclaration): + + def __init__(self, vartype): + _file = _name = _parent = None + data = vartype + self.vartype = vartype + super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>') + + +class TypeDef(TypeDeclaration): + kind = KIND.TYPEDEF + + @classmethod + def _resolve_data(cls, data): + if not data: + raise NotImplementedError(data) + vartype = dict(data) + del vartype['storage'] + return VarType(**vartype), None + + @classmethod + def _raw_data(self, data): + # XXX finish! + return data + + @classmethod + def _format_data(cls, fmt, data, extra): + text = str(data) + if fmt in ('line', 'brief'): + yield text + elif fmt == 'full': + yield text + elif fmt == 'row': + yield text + else: + raise NotImplementedError(fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + if fmt in ('line', 'brief'): + vartype, _ = VarType.from_str(datastr) + return vartype, None + #elif fmt == 'full': + elif fmt == 'row': + vartype, _ = VarType.from_str(datastr) + return vartype, None + else: + raise NotImplementedError(fmt) + + def __init__(self, file, name, data, parent=None): + super().__init__(file, name, data, parent, _shortkey=name) + + @property + def vartype(self): + return self.data + + +class Member(namedtuple('Member', 'name vartype size')): + + @classmethod + def from_data(cls, raw, index): + name = raw.name if raw.name else index + vartype = size = None + if type(raw.data) is int: + size = raw.data + elif isinstance(raw.data, str): + size = int(raw.data) + elif raw.data: + vartype = dict(raw.data) + del vartype['storage'] + if 'size' in vartype: + size = int(vartype.pop('size')) + vartype = VarType(**vartype) + return cls(name, vartype, size) + + @classmethod + def from_str(cls, text): + name, _, vartype = text.partition(': ') + if name.startswith('#'): + name = int(name[1:]) + if vartype.isdigit(): + size = int(vartype) + vartype = None + else: + vartype, _ = VarType.from_str(vartype) + size = None + return cls(name, vartype, size) + + def __str__(self): + name = self.name if isinstance(self.name, str) else f'#{self.name}' + return f'{name}: {self.vartype or self.size}' + + +class _StructUnion(TypeDeclaration): + + @classmethod + def _resolve_data(cls, data): + if not data: + # XXX There should be some! Forward? + return None, None + return [Member.from_data(v, i) for i, v in enumerate(data)], None + + @classmethod + def _raw_data(self, data): + # XXX finish! + return data + + @classmethod + def _format_data(cls, fmt, data, extra): + if fmt in ('line', 'brief'): + members = ', '.join(f'<{m}>' for m in data) + yield f'[{members}]' + elif fmt == 'full': + for member in data: + yield f'{member}' + elif fmt == 'row': + members = ', '.join(f'<{m}>' for m in data) + yield f'[{members}]' + else: + raise NotImplementedError(fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + if fmt in ('line', 'brief'): + members = [Member.from_str(m[1:-1]) + for m in datastr[1:-1].split(', ')] + return members, None + #elif fmt == 'full': + elif fmt == 'row': + members = [Member.from_str(m.rstrip('>').lstrip('<')) + for m in datastr[1:-1].split('>, <')] + return members, None + else: + raise NotImplementedError(fmt) + + def __init__(self, file, name, data, parent=None): + super().__init__(file, name, data, parent) + + @property + def members(self): + return self.data + + +class Struct(_StructUnion): + kind = KIND.STRUCT + + +class Union(_StructUnion): + kind = KIND.UNION + + +class Enum(TypeDeclaration): + kind = KIND.ENUM + + @classmethod + def _resolve_data(cls, data): + if not data: + # XXX There should be some! Forward? + return None, None + enumerators = [e if isinstance(e, str) else e.name + for e in data] + return enumerators, None + + @classmethod + def _raw_data(self, data): + # XXX finsih! + return data + + @classmethod + def _format_data(cls, fmt, data, extra): + if fmt in ('line', 'brief'): + yield repr(data) + elif fmt == 'full': + for enumerator in data: + yield f'{enumerator}' + elif fmt == 'row': + # XXX This won't work with CSV... + yield ','.join(data) + else: + raise NotImplementedError(fmt) + + @classmethod + def _unformat_data(cls, datastr, fmt=None): + if fmt in ('line', 'brief'): + return _strutil.unrepr(datastr), None + #elif fmt == 'full': + elif fmt == 'row': + return datastr.split(','), None + else: + raise NotImplementedError(fmt) + + def __init__(self, file, name, data, parent=None): + super().__init__(file, name, data, parent) + + @property + def enumerators(self): + return self.data + + +### statements ### + +class Statement(HighlevelParsedItem): + kind = KIND.STATEMENT + + @classmethod + def _resolve_data(cls, data): + # XXX finsih! + return data, None + + @classmethod + def _raw_data(self, data): + # XXX finsih! + return data + + @classmethod + def _render_data(cls, fmt, data, extra): + # XXX Handle other formats? + return repr(data) + + @classmethod + def _parse_data(self, datastr, fmt=None): + # XXX Handle other formats? + return _strutil.unrepr(datastr), None + + def __init__(self, file, name, data, parent=None): + super().__init__(file, name, data, parent, + _shortkey=data or '', + _key=( + str(file), + file.lno, + # XXX Only one stmt per line? + ), + ) + + @property + def text(self): + return self.data + + +### + +KIND_CLASSES = {cls.kind: cls for cls in [ + Variable, + Function, + TypeDef, + Struct, + Union, + Enum, + Statement, +]} + + +def resolve_parsed(parsed): + if isinstance(parsed, HighlevelParsedItem): + return parsed + try: + cls = KIND_CLASSES[parsed.kind] + except KeyError: + raise ValueError(f'unsupported kind in {parsed!r}') + return cls.from_parsed(parsed) + + +############################# +# composite + +class Declarations: + + @classmethod + def from_decls(cls, decls): + return cls(decls) + + @classmethod + def from_parsed(cls, items): + decls = (resolve_parsed(item) + for item in items + if item.kind is not KIND.STATEMENT) + return cls.from_decls(decls) + + @classmethod + def _resolve_key(cls, raw): + if isinstance(raw, str): + raw = [raw] + elif isinstance(raw, Declaration): + raw = ( + raw.filename if cls._is_public(raw) else None, + # `raw.parent` is always None for types and functions. + raw.parent if raw.kind is KIND.VARIABLE else None, + raw.name, + ) + + extra = None + if len(raw) == 1: + name, = raw + if name: + name = str(name) + if name.endswith(('.c', '.h')): + # This is only legit as a query. + key = (name, None, None) + else: + key = (None, None, name) + else: + key = (None, None, None) + elif len(raw) == 2: + parent, name = raw + name = str(name) + if isinstance(parent, Declaration): + key = (None, parent.name, name) + elif not parent: + key = (None, None, name) + else: + parent = str(parent) + if parent.endswith(('.c', '.h')): + key = (parent, None, name) + else: + key = (None, parent, name) + else: + key, extra = raw[:3], raw[3:] + filename, funcname, name = key + filename = str(filename) if filename else None + if isinstance(funcname, Declaration): + funcname = funcname.name + else: + funcname = str(funcname) if funcname else None + name = str(name) if name else None + key = (filename, funcname, name) + return key, extra + + @classmethod + def _is_public(cls, decl): + # For .c files don't we need info from .h files to make this decision? + # XXX Check for "extern". + # For now we treat all decls a "private" (have filename set). + return False + + def __init__(self, decls): + # (file, func, name) -> decl + # "public": + # * (None, None, name) + # "private", "global": + # * (file, None, name) + # "private", "local": + # * (file, func, name) + if hasattr(decls, 'items'): + self._decls = decls + else: + self._decls = {} + self._extend(decls) + + # XXX always validate? + + def validate(self): + for key, decl in self._decls.items(): + if type(key) is not tuple or len(key) != 3: + raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})') + filename, funcname, name = key + if not name: + raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})') + elif type(name) is not str: + raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})') + # XXX Check filename type? + # XXX Check funcname type? + + if decl.kind is KIND.STATEMENT: + raise ValueError(f'expected a declaration, got {decl!r}') + + def __repr__(self): + return f'{type(self).__name__}({list(self)})' + + def __len__(self): + return len(self._decls) + + def __iter__(self): + yield from self._decls + + def __getitem__(self, key): + # XXX Be more exact for the 3-tuple case? + if type(key) not in (str, tuple): + raise KeyError(f'unsupported key {key!r}') + resolved, extra = self._resolve_key(key) + if extra: + raise KeyError(f'key must have at most 3 parts, got {key!r}') + if not resolved[2]: + raise ValueError(f'expected name in key, got {key!r}') + try: + return self._decls[resolved] + except KeyError: + if type(key) is tuple and len(key) == 3: + filename, funcname, name = key + else: + filename, funcname, name = resolved + if filename and not filename.endswith(('.c', '.h')): + raise KeyError(f'invalid filename in key {key!r}') + elif funcname and funcname.endswith(('.c', '.h')): + raise KeyError(f'invalid funcname in key {key!r}') + elif name and name.endswith(('.c', '.h')): + raise KeyError(f'invalid name in key {key!r}') + else: + raise # re-raise + + @property + def types(self): + return self._find(kind=KIND.TYPES) + + @property + def functions(self): + return self._find(None, None, None, KIND.FUNCTION) + + @property + def variables(self): + return self._find(None, None, None, KIND.VARIABLE) + + def iter_all(self): + yield from self._decls.values() + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + #def add_decl(self, decl, key=None): + # decl = _resolve_parsed(decl) + # self._add_decl(decl, key) + + def find(self, *key, **explicit): + if not key: + if not explicit: + return iter(self) + return self._find(**explicit) + + resolved, extra = self._resolve_key(key) + filename, funcname, name = resolved + if not extra: + kind = None + elif len(extra) == 1: + kind, = extra + else: + raise KeyError(f'key must have at most 4 parts, got {key!r}') + + implicit= {} + if filename: + implicit['filename'] = filename + if funcname: + implicit['funcname'] = funcname + if name: + implicit['name'] = name + if kind: + implicit['kind'] = kind + return self._find(**implicit, **explicit) + + def _find(self, filename=None, funcname=None, name=None, kind=None): + for decl in self._decls.values(): + if filename and decl.filename != filename: + continue + if funcname: + if decl.kind is not KIND.VARIABLE: + continue + if decl.parent.name != funcname: + continue + if name and decl.name != name: + continue + if kind: + kinds = KIND.resolve_group(kind) + if decl.kind not in kinds: + continue + yield decl + + def _add_decl(self, decl, key=None): + if key: + if type(key) not in (str, tuple): + raise NotImplementedError((key, decl)) + # Any partial key will be turned into a full key, but that + # same partial key will still match a key lookup. + resolved, _ = self._resolve_key(key) + if not resolved[2]: + raise ValueError(f'expected name in key, got {key!r}') + key = resolved + # XXX Also add with the decl-derived key if not the same? + else: + key, _ = self._resolve_key(decl) + self._decls[key] = decl + + def _extend(self, decls): + decls = iter(decls) + # Check only the first item. + for decl in decls: + if isinstance(decl, Declaration): + self._add_decl(decl) + # Add the rest without checking. + for decl in decls: + self._add_decl(decl) + elif isinstance(decl, HighlevelParsedItem): + raise NotImplementedError(decl) + else: + try: + key, decl = decl + except ValueError: + raise NotImplementedError(decl) + if not isinstance(decl, Declaration): + raise NotImplementedError(decl) + self._add_decl(decl, key) + # Add the rest without checking. + for key, decl in decls: + self._add_decl(decl, key) + # The iterator will be exhausted at this point. |