diff options
author | Daniel Holth <dholth@fastmail.fm> | 2012-08-25 15:26:08 -0400 |
---|---|---|
committer | Daniel Holth <dholth@fastmail.fm> | 2012-08-25 15:26:08 -0400 |
commit | d60024c6b1c19620e48bca0c6645abceb7f29ba4 (patch) | |
tree | 08be8546860bd4ea9289348f2d429bf1d0dea7af /_markerlib | |
parent | add715972b27f3a584cf50d92994dfb2ffb569ed (diff) | |
download | python-setuptools-bitbucket-d60024c6b1c19620e48bca0c6645abceb7f29ba4.tar.gz |
add markerlib as _markerlib
Diffstat (limited to '_markerlib')
-rw-r--r-- | _markerlib/__init__.py | 1 | ||||
-rw-r--r-- | _markerlib/_markers_ast.py | 132 | ||||
-rw-r--r-- | _markerlib/markers.py | 110 | ||||
-rw-r--r-- | _markerlib/test_markerlib.py | 90 |
4 files changed, 333 insertions, 0 deletions
diff --git a/_markerlib/__init__.py b/_markerlib/__init__.py new file mode 100644 index 00000000..ef8c994c --- /dev/null +++ b/_markerlib/__init__.py @@ -0,0 +1 @@ +from _markerlib.markers import default_environment, compile, interpret, as_function diff --git a/_markerlib/_markers_ast.py b/_markerlib/_markers_ast.py new file mode 100644 index 00000000..b7a5e0b9 --- /dev/null +++ b/_markerlib/_markers_ast.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +""" +Just enough of ast.py for markers.py +""" + +from _ast import AST, PyCF_ONLY_AST + +def parse(source, filename='<unknown>', mode='exec'): + """ + Parse the source into an AST node. + Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). + """ + return compile(source, filename, mode, PyCF_ONLY_AST) + +def copy_location(new_node, old_node): + """ + Copy source location (`lineno` and `col_offset` attributes) from + *old_node* to *new_node* if possible, and return *new_node*. + """ + for attr in 'lineno', 'col_offset': + if attr in old_node._attributes and attr in new_node._attributes \ + and hasattr(old_node, attr): + setattr(new_node, attr, getattr(old_node, attr)) + return new_node + +def iter_fields(node): + """ + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + """ + for field in node._fields: + try: + yield field, getattr(node, field) + except AttributeError: + pass + +class NodeVisitor(object): + """ + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the `visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + Per default the visitor functions for the nodes are ``'visit_'`` + + class name of the node. So a `TryFinally` node visit function would + be `visit_TryFinally`. This behavior can be changed by overriding + the `visit` method. If no visitor function exists for a node + (return value `None`) the `generic_visit` visitor is used instead. + + Don't use the `NodeVisitor` if you want to apply changes to nodes during + traversing. For this a special visitor exists (`NodeTransformer`) that + allows modifications. + """ + + def visit(self, node): + """Visit a node.""" + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node) + +# def generic_visit(self, node): +# """Called if no explicit visitor function exists for a node.""" +# for field, value in iter_fields(node): +# if isinstance(value, list): +# for item in value: +# if isinstance(item, AST): +# self.visit(item) +# elif isinstance(value, AST): +# self.visit(value) + + +class NodeTransformer(NodeVisitor): + """ + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor methods to replace or remove the old node. If the return value of + the visitor method is ``None``, the node will be removed from its location, + otherwise it is replaced with the return value. The return value may be the + original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return copy_location(Subscript( + value=Name(id='data', ctx=Load()), + slice=Index(value=Str(s=node.id)), + ctx=node.ctx + ), node) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + """ + + def generic_visit(self, node): + for field, old_value in iter_fields(node): + old_value = getattr(node, field, None) + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, AST): + value = self.visit(value) + if value is None: + continue + elif not isinstance(value, AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, AST): + new_node = self.visit(old_value) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node +
\ No newline at end of file diff --git a/_markerlib/markers.py b/_markerlib/markers.py new file mode 100644 index 00000000..293adf72 --- /dev/null +++ b/_markerlib/markers.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Interpret PEP 345 environment markers. + +EXPR [in|==|!=|not in] EXPR [or|and] ... + +where EXPR belongs to any of those: + + python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1]) + python_full_version = sys.version.split()[0] + os.name = os.name + sys.platform = sys.platform + platform.version = platform.version() + platform.machine = platform.machine() + platform.python_implementation = platform.python_implementation() + a free string, like '2.4', or 'win32' +""" + +__all__ = ['default_environment', 'compile', 'interpret'] + +# Would import from ast but for Python 2.5 +from _ast import Compare, BoolOp, Attribute, Name, Load, Str, cmpop, boolop +try: + from ast import parse, copy_location, NodeTransformer +except ImportError: # pragma no coverage + from markerlib._markers_ast import parse, copy_location, NodeTransformer + +import os +import platform +import sys +import weakref + +_builtin_compile = compile + +from platform import python_implementation + +# restricted set of variables +_VARS = {'sys.platform': sys.platform, + 'python_version': '%s.%s' % sys.version_info[:2], + # FIXME parsing sys.platform is not reliable, but there is no other + # way to get e.g. 2.7.2+, and the PEP is defined with sys.version + 'python_full_version': sys.version.split(' ', 1)[0], + 'os.name': os.name, + 'platform.version': platform.version(), + 'platform.machine': platform.machine(), + 'platform.python_implementation': python_implementation(), + 'extra': None # wheel extension + } + +def default_environment(): + """Return copy of default PEP 385 globals dictionary.""" + return dict(_VARS) + +class ASTWhitelist(NodeTransformer): + def __init__(self, statement): + self.statement = statement # for error messages + + ALLOWED = (Compare, BoolOp, Attribute, Name, Load, Str, cmpop, boolop) + + def visit(self, node): + """Ensure statement only contains allowed nodes.""" + if not isinstance(node, self.ALLOWED): + raise SyntaxError('Not allowed in environment markers.\n%s\n%s' % + (self.statement, + (' ' * node.col_offset) + '^')) + return NodeTransformer.visit(self, node) + + def visit_Attribute(self, node): + """Flatten one level of attribute access.""" + new_node = Name("%s.%s" % (node.value.id, node.attr), node.ctx) + return copy_location(new_node, node) + +def parse_marker(marker): + tree = parse(marker, mode='eval') + new_tree = ASTWhitelist(marker).generic_visit(tree) + return new_tree + +def compile_marker(parsed_marker): + return _builtin_compile(parsed_marker, '<environment marker>', 'eval', + dont_inherit=True) + +_cache = weakref.WeakValueDictionary() + +def compile(marker): + """Return compiled marker as a function accepting an environment dict.""" + try: + return _cache[marker] + except KeyError: + pass + if not marker.strip(): + def marker_fn(environment=None, override=None): + """""" + return True + else: + compiled_marker = compile_marker(parse_marker(marker)) + def marker_fn(environment=None, override=None): + """override updates environment""" + if override is None: + override = {} + if environment is None: + environment = default_environment() + environment.update(override) + return eval(compiled_marker, environment) + marker_fn.__doc__ = marker + _cache[marker] = marker_fn + return _cache[marker] + +as_function = compile # bw compat + +def interpret(marker, environment=None): + return compile(marker)(environment) diff --git a/_markerlib/test_markerlib.py b/_markerlib/test_markerlib.py new file mode 100644 index 00000000..ff78d672 --- /dev/null +++ b/_markerlib/test_markerlib.py @@ -0,0 +1,90 @@ +import os +import unittest +import pkg_resources +from setuptools.tests.py26compat import skipIf +from unittest import expectedFailure + +try: + import _ast +except ImportError: + pass + +class TestMarkerlib(unittest.TestCase): + + def test_markers(self): + from _markerlib import interpret, default_environment, compile + + os_name = os.name + + self.assert_(interpret("")) + + self.assert_(interpret("os.name != 'buuuu'")) + self.assert_(interpret("python_version > '1.0'")) + self.assert_(interpret("python_version < '5.0'")) + self.assert_(interpret("python_version <= '5.0'")) + self.assert_(interpret("python_version >= '1.0'")) + self.assert_(interpret("'%s' in os.name" % os_name)) + self.assert_(interpret("'buuuu' not in os.name")) + + self.assertFalse(interpret("os.name == 'buuuu'")) + self.assertFalse(interpret("python_version < '1.0'")) + self.assertFalse(interpret("python_version > '5.0'")) + self.assertFalse(interpret("python_version >= '5.0'")) + self.assertFalse(interpret("python_version <= '1.0'")) + self.assertFalse(interpret("'%s' not in os.name" % os_name)) + self.assertFalse(interpret("'buuuu' in os.name and python_version >= '5.0'")) + + environment = default_environment() + environment['extra'] = 'test' + self.assert_(interpret("extra == 'test'", environment)) + self.assertFalse(interpret("extra == 'doc'", environment)) + + @expectedFailure(NameError) + def raises_nameError(): + interpret("python.version == '42'") + + raises_nameError() + + @expectedFailure(SyntaxError) + def raises_syntaxError(): + interpret("(x for x in (4,))") + + raises_syntaxError() + + statement = "python_version == '5'" + self.assertEqual(compile(statement).__doc__, statement) + + @skipIf('_ast' not in globals(), + "ast not available (Python < 2.5?)") + def test_ast(self): + try: + import ast, nose + raise nose.SkipTest() + except ImportError: + pass + + # Nonsensical code coverage tests. + import _markerlib._markers_ast as _markers_ast + + class Node(_ast.AST): + _fields = ('bogus') + list(_markers_ast.iter_fields(Node())) + + class Node2(_ast.AST): + def __init__(self): + self._fields = ('bogus',) + self.bogus = [Node()] + + class NoneTransformer(_markers_ast.NodeTransformer): + def visit_Attribute(self, node): + return None + + def visit_Str(self, node): + return None + + def visit_Node(self, node): + return [] + + NoneTransformer().visit(_markers_ast.parse('a.b = "c"')) + NoneTransformer().visit(Node2()) + |