summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Doc/whatsnew/whatsnew24.tex78
-rw-r--r--Grammar/Grammar9
-rw-r--r--Include/graminit.h38
-rw-r--r--Include/symtable.h5
-rw-r--r--Lib/compiler/ast.py76
-rw-r--r--Lib/compiler/pycodegen.py89
-rw-r--r--Lib/compiler/symbols.py41
-rw-r--r--Lib/compiler/transformer.py53
-rwxr-xr-xLib/symbol.py38
-rw-r--r--Lib/test/test_deque.py2
-rw-r--r--Lib/test/test_genexps.py258
-rw-r--r--Lib/test/test_grammar.py43
-rw-r--r--Lib/test/test_parser.py2
-rw-r--r--Misc/ACKS1
-rw-r--r--Misc/NEWS2
-rw-r--r--Modules/parsermodule.c121
-rw-r--r--Python/compile.c283
-rw-r--r--Python/graminit.c703
-rw-r--r--Python/symtable.c2
-rw-r--r--Tools/compiler/ast.txt11
20 files changed, 1503 insertions, 352 deletions
diff --git a/Doc/whatsnew/whatsnew24.tex b/Doc/whatsnew/whatsnew24.tex
index bf2c8cbf98..c60f28a078 100644
--- a/Doc/whatsnew/whatsnew24.tex
+++ b/Doc/whatsnew/whatsnew24.tex
@@ -89,6 +89,84 @@ Greg Wilson and ultimately implemented by Raymond Hettinger.}
XXX write this.
%======================================================================
+\section{PEP 229: Generator Expressions}
+
+Generator expressions create in-line generators using a syntax similar
+to list comprehensions but with parenthesis instead of the surrounding
+brackets.
+
+Genexps allow simple generators to be constructed without a separate function
+definition. Writing:
+
+\begin{verbatim}
+ g = (tgtexp for var1 in exp1 for var2 in exp2 if exp3)
+\end{verbatim}
+
+is equivalent to:
+
+\begin{verbatim}
+ def _generator(exp):
+ for var1 in exp:
+ for var2 in exp2:
+ if exp3:
+ yield tgtexp
+ g = _generator(exp1)
+ del _generator
+\end{verbatim}
+
+The advantage over full generator definitions is in economy of
+expression. Their advantage over list comprehensions is in saving
+memory by creating data only when it is needed rather than forming
+a whole list is memory all at once. Applications using memory
+friendly generator expressions may scale-up to high volumes of data
+more readily than with list comprehensions.
+
+Generator expressions are intended to be used inside functions
+such as \function{sum()}, \function{min()}, \function{set()}, and
+\function{dict()}. These functions consume their data all at once
+and would not benefit from having a full list instead of a generator
+an input:
+
+\begin{verbatim}
+>>> sum(i*i for i in range(10))
+285
+
+>>> sorted(set(i*i for i in xrange(-10, 11)))
+[0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]
+
+>>> words = "Adam apple baker Bill Nancy NASA nut".split()
+>>> dict((word.lower(), word) for word in words)
+{'apple': 'apple', 'baker': 'baker', 'bill': 'Bill', 'nasa': 'NASA',
+ 'adam': 'Adam', 'nancy': 'Nancy', 'nut': 'nut'}
+
+>>> xvec = [10, 20, 30]
+>>> yvec = [7, 5, 3]
+>>> sum(x*y for x,y in itertools.izip(xvec, yvec)) # dot product
+260
+
+\end{verbatim}
+
+These examples show the intended use for generator expressions
+in situations where the values get consumed immediately after the
+generator is created. In these situations, they operate like
+memory efficient versions of list comprehensions.
+
+For more complex uses of generators, it is strongly recommended that
+the traditional full generator definitions be used instead. In a
+generator expression, the first for-loop expression is evaluated
+as soon as the expression is defined while the other expressions do
+not get evaluated until the generator is run. This nuance is never
+an issue when the generator is used immediately. If it is not used
+right away, then it is better to write a full generator definition
+which more clearly reveals when the expressions are evaluated and is
+more obvious about the visibility and lifetime of its looping variables.
+
+\begin{seealso}
+\seepep{289}{Generator Expressions}{Proposed by Raymond Hettinger and
+implemented by Jiwon Seo with early efforts steered by Hye-Shik Chang.}
+\end{seealso}
+
+%======================================================================
\section{PEP 322: Reverse Iteration}
A new built-in function, \function{reversed(\var{seq})}, takes a sequence
diff --git a/Grammar/Grammar b/Grammar/Grammar
index e52a544bc8..ce75ba8014 100644
--- a/Grammar/Grammar
+++ b/Grammar/Grammar
@@ -80,8 +80,9 @@ arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
-atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+
+atom: '(' [testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+
listmaker: test ( list_for | (',' test)* [','] )
+testlist_gexp: test ( gen_for | (',' test)* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
@@ -95,12 +96,16 @@ dictmaker: test ':' test (',' test ':' test)* [',']
classdef: 'class' NAME ['(' testlist ')'] ':' suite
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
-argument: [test '='] test # Really [keyword '='] test
+argument: [test '='] test [gen_for] # Really [keyword '='] test
list_iter: list_for | list_if
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_if: 'if' test [list_iter]
+gen_iter: gen_for | gen_if
+gen_for: 'for' exprlist 'in' test [gen_iter]
+gen_if: 'if' test [gen_iter]
+
testlist1: test (',' test)*
# not used in grammar, but may appear in "node" passed from Parser to Compiler
diff --git a/Include/graminit.h b/Include/graminit.h
index 1f2ab3eadc..7d4a97a794 100644
--- a/Include/graminit.h
+++ b/Include/graminit.h
@@ -49,20 +49,24 @@
#define power 304
#define atom 305
#define listmaker 306
-#define lambdef 307
-#define trailer 308
-#define subscriptlist 309
-#define subscript 310
-#define sliceop 311
-#define exprlist 312
-#define testlist 313
-#define testlist_safe 314
-#define dictmaker 315
-#define classdef 316
-#define arglist 317
-#define argument 318
-#define list_iter 319
-#define list_for 320
-#define list_if 321
-#define testlist1 322
-#define encoding_decl 323
+#define testlist_gexp 307
+#define lambdef 308
+#define trailer 309
+#define subscriptlist 310
+#define subscript 311
+#define sliceop 312
+#define exprlist 313
+#define testlist 314
+#define testlist_safe 315
+#define dictmaker 316
+#define classdef 317
+#define arglist 318
+#define argument 319
+#define list_iter 320
+#define list_for 321
+#define list_if 322
+#define gen_iter 323
+#define gen_for 324
+#define gen_if 325
+#define testlist1 326
+#define encoding_decl 327
diff --git a/Include/symtable.h b/Include/symtable.h
index d2438921b0..628c3e6db2 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -46,7 +46,7 @@ typedef struct _symtable_entry {
including free refs to globals */
int ste_generator; /* true if namespace is a generator */
int ste_opt_lineno; /* lineno of last exec or import * */
- int ste_tmpname; /* temporary name counter */
+ int ste_tmpname; /* temporary name counter */
struct symtable *ste_table;
} PySymtableEntryObject;
@@ -93,6 +93,9 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define OPT_EXEC 2
#define OPT_BARE_EXEC 4
+#define GENERATOR 1
+#define GENERATOR_EXPRESSION 2
+
#ifdef __cplusplus
}
#endif
diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index a10225bbbc..064df50ed5 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -1236,6 +1236,82 @@ class ListCompFor(Node):
def __repr__(self):
return "ListCompFor(%s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.ifs))
+class GenExpr(Node):
+ nodes["genexpr"] = "GenExpr"
+ def __init__(self, code):
+ self.code = code
+ self.argnames = ['[outmost-iterable]']
+ self.varargs = self.kwargs = None
+
+ def getChildren(self):
+ return self.code,
+
+ def getChildNodes(self):
+ return self.code,
+
+ def __repr__(self):
+ return "GenExpr(%s)" % (repr(self.code),)
+
+class GenExprInner(Node):
+ nodes["genexprinner"] = "GenExprInner"
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprInner(%s, %s)" % (repr(self.expr), repr(self.quals))
+
+class GenExprFor(Node):
+ nodes["genexprfor"] = "GenExprFor"
+ def __init__(self, assign, iter, ifs):
+ self.assign = assign
+ self.iter = iter
+ self.ifs = ifs
+ self.is_outmost = False
+
+ def getChildren(self):
+ children = []
+ children.append(self.assign)
+ children.append(self.iter)
+ children.extend(flatten(self.ifs))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.assign)
+ nodelist.append(self.iter)
+ nodelist.extend(flatten_nodes(self.ifs))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprFor(%s, %s, %s)" % (repr(self.assign), repr(self.iter), repr(self.ifs))
+
+class GenExprIf(Node):
+ nodes["genexprif"] = "GenExprIf"
+ def __init__(self, test):
+ self.test = test
+
+ def getChildren(self):
+ return self.test,
+
+ def getChildNodes(self):
+ return self.test,
+
+ def __repr__(self):
+ return "GenExprIf(%s)" % (repr(self.test),)
+
klasses = globals()
for k in nodes.keys():
nodes[k] = klasses[nodes[k]]
diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index 292e859a95..ef5a0d36e9 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -619,6 +619,79 @@ class CodeGenerator:
self.newBlock()
self.emit('POP_TOP')
+ def visitGenExpr(self, node):
+ gen = GenExprCodeGenerator(node, self.scopes, self.class_name,
+ self.get_module())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno(node)
+ frees = gen.scope.get_free_vars()
+ if frees:
+ for name in frees:
+ self.emit('LOAD_CLOSURE', name)
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_CLOSURE', 0)
+ else:
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_FUNCTION', 0)
+
+ # precomputation of outmost iterable
+ self.visit(node.code.quals[0].iter)
+ self.emit('GET_ITER')
+ self.emit('CALL_FUNCTION', 1)
+
+ def visitGenExprInner(self, node):
+ self.set_lineno(node)
+ # setup list
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node.expr)
+ self.emit('YIELD_VALUE')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP_FORWARD', skip_one)
+ self.startBlock(cont)
+ self.emit('POP_TOP')
+ self.nextBlock(skip_one)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+ self.emit('LOAD_CONST', None)
+
+ def visitGenExprFor(self, node):
+ start = self.newBlock()
+ anchor = self.newBlock()
+
+ if node.is_outmost:
+ self.loadName('[outmost-iterable]')
+ else:
+ self.visit(node.iter)
+ self.emit('GET_ITER')
+
+ self.nextBlock(start)
+ self.set_lineno(node, force=True)
+ self.emit('FOR_ITER', anchor)
+ self.nextBlock()
+ self.visit(node.assign)
+ return start, anchor
+
+ def visitGenExprIf(self, node, branch):
+ self.set_lineno(node, force=True)
+ self.visit(node.test)
+ self.emit('JUMP_IF_FALSE', branch)
+ self.newBlock()
+ self.emit('POP_TOP')
+
# exception related
def visitAssert(self, node):
@@ -1199,6 +1272,7 @@ class AbstractFunctionCode:
klass.lambdaCount = klass.lambdaCount + 1
else:
name = func.name
+
args, hasTupleArg = generateArgList(func.argnames)
self.graph = pyassem.PyFlowGraph(name, func.filename, args,
optimized=1)
@@ -1263,6 +1337,21 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
if self.scope.generator is not None:
self.graph.setFlag(CO_GENERATOR)
+class GenExprCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
+ CodeGenerator):
+ super_init = CodeGenerator.__init__ # call be other init
+ scopes = None
+
+ __super_init = AbstractFunctionCode.__init__
+
+ def __init__(self, gexp, scopes, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[gexp]
+ self.__super_init(gexp, scopes, 1, class_name, mod)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+ self.graph.setFlag(CO_GENERATOR)
+
class AbstractClassCode:
def __init__(self, klass, scopes, module):
diff --git a/Lib/compiler/symbols.py b/Lib/compiler/symbols.py
index fa668f1da7..6843e19ca3 100644
--- a/Lib/compiler/symbols.py
+++ b/Lib/compiler/symbols.py
@@ -179,6 +179,21 @@ class ModuleScope(Scope):
class FunctionScope(Scope):
pass
+class GenExprScope(Scope):
+ __super_init = Scope.__init__
+
+ __counter = 1
+
+ def __init__(self, module, klass=None):
+ i = self.__counter
+ self.__counter += 1
+ self.__super_init("generator expression<%d>"%i, module, klass)
+ self.add_param('[outmost-iterable]')
+
+ def get_names(self):
+ keys = Scope.get_names()
+ return keys
+
class LambdaScope(FunctionScope):
__super_init = Scope.__init__
@@ -220,6 +235,32 @@ class SymbolVisitor:
self.visit(node.code, scope)
self.handle_free_vars(scope, parent)
+ def visitGenExpr(self, node, parent):
+ scope = GenExprScope(self.module, self.klass);
+ if parent.nested or isinstance(parent, FunctionScope) \
+ or isinstance(parent, GenExprScope):
+ scope.nested = 1
+
+ self.scopes[node] = scope
+ self.visit(node.code, scope)
+
+ self.handle_free_vars(scope, parent)
+
+ def visitGenExprInner(self, node, scope):
+ for genfor in node.quals:
+ self.visit(genfor, scope)
+
+ self.visit(node.expr, scope)
+
+ def visitGenExprFor(self, node, scope):
+ self.visit(node.assign, scope, 1)
+ self.visit(node.iter, scope)
+ for if_ in node.ifs:
+ self.visit(if_, scope)
+
+ def visitGenExprIf(self, node, scope):
+ self.visit(node.test, scope)
+
def visitLambda(self, node, parent, assign=0):
# Lambda is an expression, so it could appear in an expression
# context where assign is passed. The transformer should catch
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index 9074fc767a..6832cf1003 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -534,6 +534,12 @@ class Transformer:
testlist1 = testlist
exprlist = testlist
+ def testlist_gexp(self, nodelist):
+ if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for:
+ test = self.com_node(nodelist[0])
+ return self.com_generator_expression(test, nodelist[1])
+ return self.testlist(nodelist)
+
def test(self, nodelist):
# and_test ('or' and_test)* | lambdef
if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
@@ -1085,6 +1091,48 @@ class Transformer:
values.append(self.com_node(nodelist[i]))
return List(values)
+ if hasattr(symbol, 'gen_for'):
+ def com_generator_expression(self, expr, node):
+ # gen_iter: gen_for | gen_if
+ # gen_for: 'for' exprlist 'in' test [gen_iter]
+ # gen_if: 'if' test [gen_iter]
+
+ lineno = node[1][2]
+ fors = []
+ while node:
+ t = node[1][1]
+ if t == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ genNode = self.com_node(node[4])
+ newfor = GenExprFor(assignNode, genNode, [])
+ newfor.lineno = node[1][2]
+ fors.append(newfor)
+ if (len(node)) == 5:
+ node = None
+ else:
+ node = self.com_gen_iter(node[5])
+ elif t == 'if':
+ test = self.com_node(node[2])
+ newif = GenExprIf(test)
+ newif.lineno = node[1][2]
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ else:
+ node = self.com_gen_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected generator expression element: %s %d"
+ % (node, lineno))
+ fors[0].is_outmost = True
+ n = GenExpr(GenExprInner(expr, fors))
+ n.lineno = lineno
+ return n
+
+ def com_gen_iter(self, node):
+ assert node[0] == symbol.gen_iter
+ return node[1]
+
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
items = []
@@ -1122,6 +1170,8 @@ class Transformer:
if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
break
kw, result = self.com_argument(node, kw)
+ if len_nodelist != 2 and isinstance(result, GenExpr):
+ raise SyntaxError, 'generator expression needs parenthesis'
args.append(result)
else:
# No broken by star arg, so skip the last one we processed.
@@ -1148,6 +1198,9 @@ class Transformer:
return CallFunc(primaryNode, args, star_node, dstar_node)
def com_argument(self, nodelist, kw):
+ if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for:
+ test = self.com_node(nodelist[1])
+ return 0, self.com_generator_expression(test, nodelist[2])
if len(nodelist) == 2:
if kw:
raise SyntaxError, "non-keyword arg after keyword arg"
diff --git a/Lib/symbol.py b/Lib/symbol.py
index 38c6193179..c839e4ab46 100755
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -61,23 +61,27 @@ factor = 303
power = 304
atom = 305
listmaker = 306
-lambdef = 307
-trailer = 308
-subscriptlist = 309
-subscript = 310
-sliceop = 311
-exprlist = 312
-testlist = 313
-testlist_safe = 314
-dictmaker = 315
-classdef = 316
-arglist = 317
-argument = 318
-list_iter = 319
-list_for = 320
-list_if = 321
-testlist1 = 322
-encoding_decl = 323
+testlist_gexp = 307
+lambdef = 308
+trailer = 309
+subscriptlist = 310
+subscript = 311
+sliceop = 312
+exprlist = 313
+testlist = 314
+testlist_safe = 315
+dictmaker = 316
+classdef = 317
+arglist = 318
+argument = 319
+list_iter = 320
+list_for = 321
+list_if = 322
+gen_iter = 323
+gen_for = 324
+gen_if = 325
+testlist1 = 326
+encoding_decl = 327
#--end constants--
sym_name = {}
diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index 9b857c5d47..bad885761e 100644
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -566,7 +566,7 @@ def test_main(verbose=None):
# doctests
from test import test_deque
-# test_support.run_doctest(test_deque, verbose)
+ test_support.run_doctest(test_deque, verbose)
if __name__ == "__main__":
test_main(verbose=True)
diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py
new file mode 100644
index 0000000000..b09fc952bf
--- /dev/null
+++ b/Lib/test/test_genexps.py
@@ -0,0 +1,258 @@
+doctests = """
+
+Test simple loop with conditional
+
+ >>> sum(i*i for i in range(100) if i&1 == 1)
+ 166650
+
+Test simple nesting
+
+ >>> list((i,j) for i in range(3) for j in range(4) )
+ [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
+
+Test nesting with the inner expression dependent on the outer
+
+ >>> list((i,j) for i in range(4) for j in range(i) )
+ [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
+
+Make sure the induction variable is not exposed
+
+ >>> i = 20
+ >>> sum(i*i for i in range(100))
+ 328350
+ >>> i
+ 20
+
+Test first class
+
+ >>> g = (i*i for i in range(4))
+ >>> type(g)
+ <type 'generator'>
+ >>> list(g)
+ [0, 1, 4, 9]
+
+Test direct calls to next()
+
+ >>> g = (i*i for i in range(3))
+ >>> g.next()
+ 0
+ >>> g.next()
+ 1
+ >>> g.next()
+ 4
+ >>> g.next()
+ Traceback (most recent call last):
+ File "<pyshell#21>", line 1, in -toplevel-
+ g.next()
+ StopIteration
+
+Does it stay stopped?
+
+ >>> g.next()
+ Traceback (most recent call last):
+ File "<pyshell#21>", line 1, in -toplevel-
+ g.next()
+ StopIteration
+ >>> list(g)
+ []
+
+Test running gen when defining function is out of scope
+
+ >>> def f(n):
+ ... return (i*i for i in xrange(n))
+ ...
+ >>> list(f(10))
+ [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
+
+ >>> def f(n):
+ ... return ((i,j) for i in xrange(3) for j in xrange(n))
+ ...
+ >>> list(f(4))
+ [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
+ >>> def f(n):
+ ... return ((i,j) for i in xrange(3) for j in xrange(4) if j in xrange(n))
+ ...
+ >>> list(f(4))
+ [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
+ >>> list(f(2))
+ [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0), (2, 1)]
+
+#Verify that parenthesis are required in a statement
+#>>> def f(n):
+#... return i*i for i in xrange(n)
+#...
+#SyntaxError: invalid syntax
+
+Verify early binding for the outermost for-expression
+
+ >>> x=10
+ >>> g = (i*i for i in range(x))
+ >>> x = 5
+ >>> list(g)
+ [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
+
+Verify late binding for the outermost if-expression
+
+ >>> include = (2,4,6,8)
+ >>> g = (i*i for i in range(10) if i in include)
+ >>> include = (1,3,5,7,9)
+ >>> list(g)
+ [1, 9, 25, 49, 81]
+
+Verify late binding for the innermost for-expression
+
+ >>> g = ((i,j) for i in range(3) for j in range(x))
+ >>> x = 4
+ >>> list(g)
+ [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
+
+Verify re-use of tuples (a side benefit of using genexps over listcomps)
+
+ >>> tupleids = map(id, ((i,i) for i in xrange(10)))
+ >>> max(tupleids) - min(tupleids)
+ 0
+
+
+
+########### Tests borrowed from or inspired by test_generators.py ############
+
+Make a generator that acts like range()
+
+ >>> yrange = lambda n: (i for i in xrange(n))
+ >>> list(yrange(10))
+ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+Generators always return to the most recent caller:
+
+ >>> def creator():
+ ... r = yrange(5)
+ ... print "creator", r.next()
+ ... return r
+ ...
+ >>> def caller():
+ ... r = creator()
+ ... for i in r:
+ ... print "caller", i
+ ...
+ >>> caller()
+ creator 0
+ caller 1
+ caller 2
+ caller 3
+ caller 4
+
+Generators can call other generators:
+
+ >>> def zrange(n):
+ ... for i in yrange(n):
+ ... yield i
+ ...
+ >>> list(zrange(5))
+ [0, 1, 2, 3, 4]
+
+
+Verify that a gen exp cannot be resumed while it is actively running:
+
+ >>> g = (me.next() for i in xrange(10))
+ >>> me = g
+ >>> me.next()
+ Traceback (most recent call last):
+ File "<pyshell#30>", line 1, in -toplevel-
+ me.next()
+ File "<pyshell#28>", line 1, in <generator expression>
+ g = (me.next() for i in xrange(10))
+ ValueError: generator already executing
+
+Verify exception propagation
+
+ >>> g = (10 // i for i in (5, 0, 2))
+ >>> g.next()
+ 2
+ >>> g.next()
+ Traceback (most recent call last):
+ File "<pyshell#37>", line 1, in -toplevel-
+ g.next()
+ File "<pyshell#35>", line 1, in <generator expression>
+ g = (10 // i for i in (5, 0, 2))
+ ZeroDivisionError: integer division or modulo by zero
+ >>> g.next()
+ Traceback (most recent call last):
+ File "<pyshell#38>", line 1, in -toplevel-
+ g.next()
+ StopIteration
+
+Make sure that None is a valid return value
+
+ >>> list(None for i in xrange(10))
+ [None, None, None, None, None, None, None, None, None, None]
+
+Check that generator attributes are present
+
+ >>> g = (i*i for i in range(3))
+ >>> expected = set(['gi_frame', 'gi_running', 'next'])
+ >>> set(attr for attr in dir(g) if not attr.startswith('__')) >= expected
+ True
+
+ >>> print g.next.__doc__
+ x.next() -> the next value, or raise StopIteration
+ >>> import types
+ >>> isinstance(g, types.GeneratorType)
+ True
+
+Check the __iter__ slot is defined to return self
+
+ >>> iter(g) is g
+ True
+
+Verify that the running flag is set properly
+
+ >>> g = (me.gi_running for i in (0,1))
+ >>> me = g
+ >>> me.gi_running
+ 0
+ >>> me.next()
+ 1
+ >>> me.gi_running
+ 0
+
+Verify that genexps are weakly referencable
+
+ >>> import weakref
+ >>> g = (i*i for i in range(4))
+ >>> wr = weakref.ref(g)
+ >>> wr() is g
+ True
+ >>> p = weakref.proxy(g)
+ >>> list(p)
+ [0, 1, 4, 9]
+
+
+"""
+
+
+__test__ = {'doctests' : doctests}
+
+def test_main(verbose=None):
+ import sys
+ from test import test_support
+ from test import test_genexps
+ test_support.run_doctest(test_genexps, verbose)
+
+ # verify reference counting
+ if verbose and hasattr(sys, "gettotalrefcount"):
+ import gc
+ counts = [None] * 5
+ for i in xrange(len(counts)):
+ test_support.run_doctest(test_genexps, verbose)
+ gc.collect()
+ counts[i] = sys.gettotalrefcount()
+ print counts
+
+if __name__ == "__main__":
+ test_main(verbose=True)
+
+
+
+
+
+
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 0eed4bbc7b..e0d5b745d6 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -739,3 +739,46 @@ print [
for (sp_sno, sp_pno) in suppart
if sno == sp_sno and pno == sp_pno
]
+
+# generator expression tests
+g = ([x for x in range(10)] for x in range(1))
+verify(g.next() == [x for x in range(10)])
+try:
+ g.next()
+ raise TestFailed, 'should produce StopIteration exception'
+except StopIteration:
+ pass
+
+a = 1
+try:
+ g = (a for d in a)
+ g.next()
+ raise TestFailed, 'should produce TypeError'
+except TypeError:
+ pass
+
+verify(list((x, y) for x in 'abcd' for y in 'abcd') == [(x, y) for x in 'abcd' for y in 'abcd'])
+verify(list((x, y) for x in 'ab' for y in 'xy') == [(x, y) for x in 'ab' for y in 'xy'])
+
+a = [x for x in range(10)]
+b = (x for x in (y for y in a))
+verify(sum(b) == sum([x for x in range(10)]))
+
+verify(sum(x**2 for x in range(10)) == sum([x**2 for x in range(10)]))
+verify(sum(x*x for x in range(10) if x%2) == sum([x*x for x in range(10) if x%2]))
+verify(sum(x for x in (y for y in range(10))) == sum([x for x in range(10)]))
+verify(sum(x for x in (y for y in (z for z in range(10)))) == sum([x for x in range(10)]))
+verify(sum(x for x in [y for y in (z for z in range(10))]) == sum([x for x in range(10)]))
+verify(sum(x for x in (y for y in (z for z in range(10) if True)) if True) == sum([x for x in range(10)]))
+verify(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True) == 0)
+check_syntax("foo(x for x in range(10), 100)")
+check_syntax("foo(100, x for x in range(10))")
+
+# test for outmost iterable precomputation
+x = 10; g = (i for i in range(x)); x = 5
+verify(len(list(g)) == 10)
+
+# This should hold, since we're only precomputing outmost iterable.
+x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+x = 5; t = True;
+verify([(i,j) for i in range(10) for j in range(5)] == list(g))
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 7860e7bf83..9652f6bd1d 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -67,6 +67,8 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
self.check_expr("lambda x, *y, **z: 0")
+ self.check_expr("(x for x in range(10))")
+ self.check_expr("foo(x for x in range(10))")
def test_print(self):
self.check_suite("print")
diff --git a/Misc/ACKS b/Misc/ACKS
index f91b65d936..fb3c94f5e2 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -500,6 +500,7 @@ Barry Scott
Steven Scott
Nick Seidenman
Fred Sells
+Jiwon Seo
Denis Severson
Ha Shao
Bruce Sherwood
diff --git a/Misc/NEWS b/Misc/NEWS
index f111d0ef4a..58ad98c55a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,8 @@ What's New in Python 2.4 alpha 1?
Core and builtins
-----------------
+- Implemented generator expressions (PEP 289). Coded by Jiwon Seo.
+
- Enabled the profiling of C extension functions (and builtins) - check
new documentation and modified profiler and bdb modules for more details
diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
index bf6a360116..35c5dee01a 100644
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -855,7 +855,9 @@ VALIDATER(subscriptlist); VALIDATER(sliceop);
VALIDATER(exprlist); VALIDATER(dictmaker);
VALIDATER(arglist); VALIDATER(argument);
VALIDATER(listmaker); VALIDATER(yield_stmt);
-VALIDATER(testlist1);
+VALIDATER(testlist1); VALIDATER(gen_for);
+VALIDATER(gen_iter); VALIDATER(gen_if);
+VALIDATER(testlist_gexp);
#undef VALIDATER
@@ -1246,6 +1248,21 @@ validate_list_iter(node *tree)
return res;
}
+/* gen_iter: gen_for | gen_if
+ */
+static int
+validate_gen_iter(node *tree)
+{
+ int res = (validate_ntype(tree, gen_iter)
+ && validate_numnodes(tree, 1, "gen_iter"));
+ if (res && TYPE(CHILD(tree, 0)) == gen_for)
+ res = validate_gen_for(CHILD(tree, 0));
+ else
+ res = validate_gen_if(CHILD(tree, 0));
+
+ return res;
+}
+
/* list_for: 'for' exprlist 'in' testlist [list_iter]
*/
static int
@@ -1268,6 +1285,28 @@ validate_list_for(node *tree)
return res;
}
+/* gen_for: 'for' exprlist 'in' test [gen_iter]
+ */
+static int
+validate_gen_for(node *tree)
+{
+ int nch = NCH(tree);
+ int res;
+
+ if (nch == 5)
+ res = validate_gen_iter(CHILD(tree, 4));
+ else
+ res = validate_numnodes(tree, 4, "gen_for");
+
+ if (res)
+ res = (validate_name(CHILD(tree, 0), "for")
+ && validate_exprlist(CHILD(tree, 1))
+ && validate_name(CHILD(tree, 2), "in")
+ && validate_test(CHILD(tree, 3)));
+
+ return res;
+}
+
/* list_if: 'if' test [list_iter]
*/
static int
@@ -1288,6 +1327,25 @@ validate_list_if(node *tree)
return res;
}
+/* gen_if: 'if' test [gen_iter]
+ */
+static int
+validate_gen_if(node *tree)
+{
+ int nch = NCH(tree);
+ int res;
+
+ if (nch == 3)
+ res = validate_gen_iter(CHILD(tree, 2));
+ else
+ res = validate_numnodes(tree, 2, "gen_if");
+
+ if (res)
+ res = (validate_name(CHILD(tree, 0), "if")
+ && validate_test(CHILD(tree, 1)));
+
+ return res;
+}
/* validate_fpdef()
*
@@ -2187,7 +2245,7 @@ validate_atom(node *tree)
&& (validate_rparen(CHILD(tree, nch - 1))));
if (res && (nch == 3))
- res = validate_testlist(CHILD(tree, 1));
+ res = validate_testlist_gexp(CHILD(tree, 1));
break;
case LSQB:
if (nch == 2)
@@ -2244,7 +2302,7 @@ validate_listmaker(node *tree)
ok = validate_test(CHILD(tree, 0));
/*
- * list_iter | (',' test)* [',']
+ * list_for | (',' test)* [',']
*/
if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for)
ok = validate_list_for(CHILD(tree, 1));
@@ -2266,6 +2324,43 @@ validate_listmaker(node *tree)
return ok;
}
+/* testlist_gexp:
+ * test ( gen_for | (',' test)* [','] )
+ */
+static int
+validate_testlist_gexp(node *tree)
+{
+ int nch = NCH(tree);
+ int ok = nch;
+
+ if (nch == 0)
+ err_string("missing child nodes of testlist_gexp");
+ else {
+ ok = validate_test(CHILD(tree, 0));
+ }
+
+ /*
+ * gen_for | (',' test)* [',']
+ */
+ if (nch == 2 && TYPE(CHILD(tree, 1)) == gen_for)
+ ok = validate_gen_for(CHILD(tree, 1));
+ else {
+ /* (',' test)* [','] */
+ int i = 1;
+ while (ok && nch - i >= 2) {
+ ok = (validate_comma(CHILD(tree, i))
+ && validate_test(CHILD(tree, i+1)));
+ i += 2;
+ }
+ if (ok && i == nch-1)
+ ok = validate_comma(CHILD(tree, i));
+ else if (i != nch) {
+ ok = 0;
+ err_string("illegal trailing nodes for testlist_gexp");
+ }
+ }
+ return ok;
+}
/* funcdef:
* 'def' NAME parameters ':' suite
@@ -2318,6 +2413,18 @@ validate_arglist(node *tree)
/* raise the right error from having an invalid number of children */
return validate_numnodes(tree, nch + 1, "arglist");
+ if (nch > 1) {
+ for (i=0; i<nch; i++) {
+ if (TYPE(CHILD(tree, i)) == argument) {
+ node *ch = CHILD(tree, i);
+ if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == gen_for) {
+ err_string("need '(', ')' for generator expression");
+ return 0;
+ }
+ }
+ }
+ }
+
while (ok && nch-i >= 2) {
/* skip leading (argument ',') */
ok = (validate_argument(CHILD(tree, i))
@@ -2377,17 +2484,19 @@ validate_arglist(node *tree)
/* argument:
*
- * [test '='] test
+ * [test '='] test [gen_for]
*/
static int
validate_argument(node *tree)
{
int nch = NCH(tree);
int res = (validate_ntype(tree, argument)
- && ((nch == 1) || (nch == 3))
+ && ((nch == 1) || (nch == 2) || (nch == 3))
&& validate_test(CHILD(tree, 0)));
- if (res && (nch == 3))
+ if (res && (nch == 2))
+ res = validate_gen_for(CHILD(tree, 1));
+ else if (res && (nch == 3))
res = (validate_equal(CHILD(tree, 1))
&& validate_test(CHILD(tree, 2)));
diff --git a/Python/compile.c b/Python/compile.c
index 7bbcd62e74..15159f840e 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -744,11 +744,15 @@ static int com_add(struct compiling *, PyObject *, PyObject *, PyObject *);
static int com_addconst(struct compiling *, PyObject *);
static int com_addname(struct compiling *, PyObject *);
static void com_addopname(struct compiling *, int, node *);
+static void com_test(struct compiling *c, node *n);
static void com_list(struct compiling *, node *, int);
static void com_list_iter(struct compiling *, node *, node *, char *);
+static void com_gen_iter(struct compiling *, node *, node *);
static int com_argdefs(struct compiling *, node *);
static void com_assign(struct compiling *, node *, int, node *);
static void com_assign_name(struct compiling *, node *, int);
+static int com_make_closure(struct compiling *c, PyCodeObject *co);
+
static PyCodeObject *icompile(node *, struct compiling *);
static PyCodeObject *jcompile(node *, const char *, struct compiling *,
PyCompilerFlags *);
@@ -759,6 +763,7 @@ static node *get_rawdocstring(node *);
static int get_ref_type(struct compiling *, char *);
/* symtable operations */
+static int symtable_lookup(struct symtable *st, char *name);
static struct symtable *symtable_build(node *, PyFutureFeatures *,
const char *filename);
static int symtable_load_symbols(struct compiling *);
@@ -777,7 +782,10 @@ static void symtable_global(struct symtable *, node *);
static void symtable_import(struct symtable *, node *);
static void symtable_assign(struct symtable *, node *, int);
static void symtable_list_comprehension(struct symtable *, node *);
+static void symtable_generator_expression(struct symtable *, node *);
static void symtable_list_for(struct symtable *, node *);
+static void symtable_gen_for(struct symtable *, node *, int);
+static void symtable_gen_iter(struct symtable *, node *);
static int symtable_update_free_vars(struct symtable *);
static int symtable_undo_free(struct symtable *, PyObject *, PyObject *);
@@ -1589,7 +1597,7 @@ com_list_for(struct compiling *c, node *n, node *e, char *t)
int anchor = 0;
int save_begin = c->c_begin;
- /* list_iter: for v in expr [list_iter] */
+ /* list_for: for v in expr [list_iter] */
com_node(c, CHILD(n, 3)); /* expr */
com_addbyte(c, GET_ITER);
c->c_begin = c->c_nexti;
@@ -1606,6 +1614,52 @@ com_list_for(struct compiling *c, node *n, node *e, char *t)
}
static void
+com_gen_for(struct compiling *c, node *n, node *t, int is_outmost)
+{
+ int break_anchor = 0;
+ int anchor = 0;
+ int save_begin = c->c_begin;
+
+ REQ(n, gen_for);
+ /* gen_for: for v in test [gen_iter] */
+
+ com_addfwref(c, SETUP_LOOP, &break_anchor);
+ block_push(c, SETUP_LOOP);
+
+ if (is_outmost) {
+ com_addop_varname(c, VAR_LOAD, "[outmost-iterable]");
+ com_push(c, 1);
+ }
+ else {
+ com_node(c, CHILD(n, 3));
+ com_addbyte(c, GET_ITER);
+ }
+
+ c->c_begin = c->c_nexti;
+ com_set_lineno(c, c->c_last_line);
+ com_addfwref(c, FOR_ITER, &anchor);
+ com_push(c, 1);
+ com_assign(c, CHILD(n, 1), OP_ASSIGN, NULL);
+
+ if (NCH(n) == 5)
+ com_gen_iter(c, CHILD(n, 4), t);
+ else {
+ com_test(c, t);
+ com_addbyte(c, YIELD_VALUE);
+ com_pop(c, 1);
+ }
+
+ com_addoparg(c, JUMP_ABSOLUTE, c->c_begin);
+ c->c_begin = save_begin;
+
+ com_backpatch(c, anchor);
+ com_pop(c, 1); /* FOR_ITER has popped this */
+ com_addbyte(c, POP_BLOCK);
+ block_pop(c, SETUP_LOOP);
+ com_backpatch(c, break_anchor);
+}
+
+static void
com_list_if(struct compiling *c, node *n, node *e, char *t)
{
int anchor = 0;
@@ -1624,6 +1678,32 @@ com_list_if(struct compiling *c, node *n, node *e, char *t)
}
static void
+com_gen_if(struct compiling *c, node *n, node *t)
+{
+ /* gen_if: 'if' test [gen_iter] */
+ int anchor = 0;
+ int a=0;
+
+ com_node(c, CHILD(n, 1));
+ com_addfwref(c, JUMP_IF_FALSE, &a);
+ com_addbyte(c, POP_TOP);
+ com_pop(c, 1);
+
+ if (NCH(n) == 3)
+ com_gen_iter(c, CHILD(n, 2), t);
+ else {
+ com_test(c, t);
+ com_addbyte(c, YIELD_VALUE);
+ com_pop(c, 1);
+ }
+ com_addfwref(c, JUMP_FORWARD, &anchor);
+ com_backpatch(c, a);
+ /* We jump here with an extra entry which we now pop */
+ com_addbyte(c, POP_TOP);
+ com_backpatch(c, anchor);
+}
+
+static void
com_list_iter(struct compiling *c,
node *p, /* parent of list_iter node */
node *e, /* element expression node */
@@ -1655,6 +1735,28 @@ com_list_iter(struct compiling *c,
}
static void
+com_gen_iter(struct compiling *c, node *n, node *t)
+{
+ /* gen_iter: gen_for | gen_if */
+ node *ch;
+ REQ(n, gen_iter);
+
+ ch = CHILD(n, 0);
+
+ switch (TYPE(ch)) {
+ case gen_for:
+ com_gen_for(c, ch, t, 0);
+ break;
+ case gen_if:
+ com_gen_if(c, ch, t);
+ break;
+ default:
+ com_error(c, PyExc_SystemError,
+ "invalid gen_iter node type");
+ }
+}
+
+static void
com_list_comprehension(struct compiling *c, node *n)
{
/* listmaker: test list_for */
@@ -1689,6 +1791,52 @@ com_listmaker(struct compiling *c, node *n)
}
static void
+com_generator_expression(struct compiling *c, node *n)
+{
+ /* testlist_gexp: test gen_for */
+ /* argument: test gen_for */
+ PyCodeObject *co;
+
+ REQ(CHILD(n, 0), test);
+ REQ(CHILD(n, 1), gen_for);
+
+ symtable_enter_scope(c->c_symtable, "<genexpr>", TYPE(n),
+ n->n_lineno);
+ co = icompile(n, c);
+ symtable_exit_scope(c->c_symtable);
+
+ if (co == NULL)
+ c->c_errors++;
+ else {
+ int closure = com_make_closure(c, co);
+ int i = com_addconst(c, (PyObject *)co);
+
+ com_addoparg(c, LOAD_CONST, i);
+ com_push(c, 1);
+ if (closure)
+ com_addoparg(c, MAKE_CLOSURE, 0);
+ else
+ com_addoparg(c, MAKE_FUNCTION, 0);
+
+ com_test(c, CHILD(CHILD(n, 1), 3));
+ com_addbyte(c, GET_ITER);
+ com_addoparg(c, CALL_FUNCTION, 1);
+ com_pop(c, 1);
+
+ Py_DECREF(co);
+ }
+}
+
+static void
+com_testlist_gexp(struct compiling *c, node *n)
+{
+ /* testlist_gexp: test ( gen_for | (',' test)* [','] ) */
+ if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for)
+ com_generator_expression(c, n);
+ else com_list(c, n, 0);
+}
+
+static void
com_dictmaker(struct compiling *c, node *n)
{
int i;
@@ -1721,7 +1869,7 @@ com_atom(struct compiling *c, node *n)
com_push(c, 1);
}
else
- com_node(c, CHILD(n, 1));
+ com_testlist_gexp(c, CHILD(n, 1));
break;
case LSQB: /* '[' [listmaker] ']' */
if (TYPE(CHILD(n, 1)) == RSQB) {
@@ -1857,7 +2005,7 @@ static void
com_argument(struct compiling *c, node *n, PyObject **pkeywords)
{
node *m;
- REQ(n, argument); /* [test '='] test; really [keyword '='] test */
+ REQ(n, argument); /* [test '='] test [gen_for]; really [keyword '='] test */
if (NCH(n) == 1) {
if (*pkeywords != NULL) {
com_error(c, PyExc_SyntaxError,
@@ -1868,6 +2016,11 @@ com_argument(struct compiling *c, node *n, PyObject **pkeywords)
}
return;
}
+ if (NCH(n) == 2) {
+ com_generator_expression(c, n);
+ return;
+ }
+
m = n;
do {
m = CHILD(m, 0);
@@ -2723,7 +2876,8 @@ static void
com_assign_sequence(struct compiling *c, node *n, int assigning)
{
int i;
- if (TYPE(n) != testlist && TYPE(n) != listmaker)
+ if (TYPE(n) != testlist && TYPE(n) != testlist_gexp &&
+ TYPE(n) != listmaker)
REQ(n, exprlist);
if (assigning) {
i = (NCH(n)+1)/2;
@@ -2765,7 +2919,13 @@ com_assign(struct compiling *c, node *n, int assigning, node *augn)
case exprlist:
case testlist:
case testlist1:
+ case testlist_gexp:
if (NCH(n) > 1) {
+ if (TYPE(CHILD(n, 1)) == gen_for) {
+ com_error(c, PyExc_SystemError,
+ "assign to generator expression not possible");
+ return;
+ }
if (assigning > OP_APPLY) {
com_error(c, PyExc_SyntaxError,
"augmented assign to tuple not possible");
@@ -4253,6 +4413,23 @@ compile_classdef(struct compiling *c, node *n)
}
static void
+compile_generator_expression(struct compiling *c, node *n)
+{
+ /* testlist_gexp: test gen_for */
+ /* argument: test gen_for */
+ REQ(CHILD(n, 0), test);
+ REQ(CHILD(n, 1), gen_for);
+
+ c->c_name = "<generator expression>";
+ com_gen_for(c, CHILD(n, 1), CHILD(n, 0), 1);
+
+ com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+ com_push(c, 1);
+ com_addbyte(c, RETURN_VALUE);
+ com_pop(c, 1);
+}
+
+static void
compile_node(struct compiling *c, node *n)
{
com_set_lineno(c, n->n_lineno);
@@ -4300,6 +4477,11 @@ compile_node(struct compiling *c, node *n)
compile_classdef(c, n);
break;
+ case testlist_gexp: /* A generator expression */
+ case argument: /* A generator expression */
+ compile_generator_expression(c, n);
+ break;
+
default:
com_error(c, PyExc_SystemError,
"compile_node: unexpected node type");
@@ -4976,7 +5158,6 @@ symtable_load_symbols(struct compiling *c)
}
}
}
-
assert(PyDict_Size(c->c_freevars) == si.si_nfrees);
if (si.si_ncells > 1) { /* one cell is always in order */
@@ -5346,11 +5527,11 @@ look_for_yield(node *n)
return 0;
case yield_stmt:
- return 1;
+ return GENERATOR;
default:
if (look_for_yield(kid))
- return 1;
+ return GENERATOR;
}
}
return 0;
@@ -5494,6 +5675,18 @@ symtable_node(struct symtable *st, node *n)
if (TYPE(CHILD(n, i)) >= single_input)
symtable_node(st, CHILD(n, i));
break;
+ case arglist:
+ if (NCH(n) > 1)
+ for (i = 0; i < NCH(n); ++i) {
+ node *ch = CHILD(n, i);
+ if (TYPE(ch) == argument && NCH(ch) == 2 &&
+ TYPE(CHILD(ch, 1)) == gen_for) {
+ PyErr_SetString(PyExc_SyntaxError,
+ "invalid syntax");
+ symtable_error(st, n->n_lineno);
+ return;
+ }
+ }
/* The remaining cases fall through to default except in
special circumstances. This requires the individual cases
to be coded with great care, even though they look like
@@ -5504,6 +5697,11 @@ symtable_node(struct symtable *st, node *n)
n = CHILD(n, 2);
goto loop;
}
+ else if (TYPE(n) == argument && NCH(n) == 2 &&
+ TYPE(CHILD(n, 1)) == gen_for) {
+ symtable_generator_expression(st, n);
+ break;
+ }
/* fall through */
case listmaker:
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == list_for) {
@@ -5511,6 +5709,13 @@ symtable_node(struct symtable *st, node *n)
break;
}
/* fall through */
+ case testlist_gexp:
+ if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) {
+ symtable_generator_expression(st, n);
+ break;
+ }
+ /* fall through */
+
case atom:
if (TYPE(n) == atom && TYPE(CHILD(n, 0)) == NAME) {
symtable_add_use(st, STR(CHILD(n, 0)));
@@ -5715,6 +5920,26 @@ symtable_list_comprehension(struct symtable *st, node *n)
}
static void
+symtable_generator_expression(struct symtable *st, node *n)
+{
+ /* testlist_gexp: test gen_for */
+ REQ(CHILD(n, 0), test);
+ REQ(CHILD(n, 1), gen_for);
+
+ symtable_enter_scope(st, "<genexpr>", TYPE(n), n->n_lineno);
+ st->st_cur->ste_generator = GENERATOR_EXPRESSION;
+
+ symtable_add_def(st, "[outmost-iterable]", DEF_PARAM);
+
+ symtable_gen_for(st, CHILD(n, 1), 1);
+ symtable_node(st, CHILD(n, 0));
+ symtable_exit_scope(st);
+
+ /* for outmost iterable precomputation */
+ symtable_node(st, CHILD(CHILD(n, 1), 3));
+}
+
+static void
symtable_list_for(struct symtable *st, node *n)
{
REQ(n, list_for);
@@ -5726,6 +5951,39 @@ symtable_list_for(struct symtable *st, node *n)
}
static void
+symtable_gen_for(struct symtable *st, node *n, int is_outmost)
+{
+ REQ(n, gen_for);
+
+ /* gen_for: for v in test [gen_iter] */
+ symtable_assign(st, CHILD(n, 1), 0);
+ if (is_outmost)
+ symtable_add_use(st, "[outmost-iterable]");
+ else
+ symtable_node(st, CHILD(n, 3));
+
+ if (NCH(n) == 5)
+ symtable_gen_iter(st, CHILD(n, 4));
+}
+
+static void
+symtable_gen_iter(struct symtable *st, node *n)
+{
+ REQ(n, gen_iter);
+
+ n = CHILD(n, 0);
+ if (TYPE(n) == gen_for)
+ symtable_gen_for(st, n, 0);
+ else {
+ REQ(n, gen_if);
+ symtable_node(st, CHILD(n, 1));
+
+ if (NCH(n) == 3)
+ symtable_gen_iter(st, CHILD(n, 2));
+ }
+}
+
+static void
symtable_import(struct symtable *st, node *n)
{
int i;
@@ -5813,6 +6071,17 @@ symtable_assign(struct symtable *st, node *n, int def_flag)
symtable_assign(st, CHILD(n, i), def_flag);
}
return;
+ case testlist_gexp:
+ if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) {
+ /* XXX This is an error, but the next pass
+ will catch it. */
+ return;
+ } else {
+ for (i = 0; i < NCH(n); i += 2)
+ symtable_assign(st, CHILD(n, i), def_flag);
+ }
+ return;
+
case exprlist:
case testlist:
case testlist1:
diff --git a/Python/graminit.c b/Python/graminit.c
index 98bad94bc7..cd8240bac1 100644
--- a/Python/graminit.c
+++ b/Python/graminit.c
@@ -1016,46 +1016,46 @@ static state states_48[4] = {
};
static arc arcs_49_0[7] = {
{16, 1},
- {127, 2},
- {130, 3},
- {133, 4},
+ {128, 2},
+ {131, 3},
+ {134, 4},
{12, 5},
- {135, 5},
- {136, 6},
+ {136, 5},
+ {137, 6},
};
static arc arcs_49_1[2] = {
- {9, 7},
+ {127, 7},
{18, 5},
};
static arc arcs_49_2[2] = {
- {128, 8},
- {129, 5},
+ {129, 8},
+ {130, 5},
};
static arc arcs_49_3[2] = {
- {131, 9},
- {132, 5},
+ {132, 9},
+ {133, 5},
};
static arc arcs_49_4[1] = {
- {134, 10},
+ {135, 10},
};
static arc arcs_49_5[1] = {
{0, 5},
};
static arc arcs_49_6[2] = {
- {136, 6},
+ {137, 6},
{0, 6},
};
static arc arcs_49_7[1] = {
{18, 5},
};
static arc arcs_49_8[1] = {
- {129, 5},
+ {130, 5},
};
static arc arcs_49_9[1] = {
- {132, 5},
+ {133, 5},
};
static arc arcs_49_10[1] = {
- {133, 5},
+ {134, 5},
};
static state states_49[11] = {
{7, arcs_49_0},
@@ -1074,7 +1074,7 @@ static arc arcs_50_0[1] = {
{21, 1},
};
static arc arcs_50_1[3] = {
- {137, 2},
+ {138, 2},
{22, 3},
{0, 1},
};
@@ -1097,153 +1097,163 @@ static state states_50[5] = {
{2, arcs_50_4},
};
static arc arcs_51_0[1] = {
- {138, 1},
+ {21, 1},
};
-static arc arcs_51_1[2] = {
- {17, 2},
- {14, 3},
+static arc arcs_51_1[3] = {
+ {139, 2},
+ {22, 3},
+ {0, 1},
};
static arc arcs_51_2[1] = {
- {14, 3},
+ {0, 2},
};
-static arc arcs_51_3[1] = {
+static arc arcs_51_3[2] = {
{21, 4},
+ {0, 3},
};
-static arc arcs_51_4[1] = {
+static arc arcs_51_4[2] = {
+ {22, 3},
{0, 4},
};
static state states_51[5] = {
{1, arcs_51_0},
- {2, arcs_51_1},
+ {3, arcs_51_1},
{1, arcs_51_2},
- {1, arcs_51_3},
- {1, arcs_51_4},
+ {2, arcs_51_3},
+ {2, arcs_51_4},
};
-static arc arcs_52_0[3] = {
- {16, 1},
- {127, 2},
- {70, 3},
+static arc arcs_52_0[1] = {
+ {140, 1},
};
static arc arcs_52_1[2] = {
- {139, 4},
- {18, 5},
+ {17, 2},
+ {14, 3},
};
static arc arcs_52_2[1] = {
- {140, 6},
+ {14, 3},
};
static arc arcs_52_3[1] = {
- {12, 5},
+ {21, 4},
};
static arc arcs_52_4[1] = {
- {18, 5},
-};
-static arc arcs_52_5[1] = {
- {0, 5},
-};
-static arc arcs_52_6[1] = {
- {129, 5},
+ {0, 4},
};
-static state states_52[7] = {
- {3, arcs_52_0},
+static state states_52[5] = {
+ {1, arcs_52_0},
{2, arcs_52_1},
{1, arcs_52_2},
{1, arcs_52_3},
{1, arcs_52_4},
- {1, arcs_52_5},
- {1, arcs_52_6},
};
-static arc arcs_53_0[1] = {
- {141, 1},
+static arc arcs_53_0[3] = {
+ {16, 1},
+ {128, 2},
+ {70, 3},
};
static arc arcs_53_1[2] = {
+ {141, 4},
+ {18, 5},
+};
+static arc arcs_53_2[1] = {
+ {142, 6},
+};
+static arc arcs_53_3[1] = {
+ {12, 5},
+};
+static arc arcs_53_4[1] = {
+ {18, 5},
+};
+static arc arcs_53_5[1] = {
+ {0, 5},
+};
+static arc arcs_53_6[1] = {
+ {130, 5},
+};
+static state states_53[7] = {
+ {3, arcs_53_0},
+ {2, arcs_53_1},
+ {1, arcs_53_2},
+ {1, arcs_53_3},
+ {1, arcs_53_4},
+ {1, arcs_53_5},
+ {1, arcs_53_6},
+};
+static arc arcs_54_0[1] = {
+ {143, 1},
+};
+static arc arcs_54_1[2] = {
{22, 2},
{0, 1},
};
-static arc arcs_53_2[2] = {
- {141, 1},
+static arc arcs_54_2[2] = {
+ {143, 1},
{0, 2},
};
-static state states_53[3] = {
- {1, arcs_53_0},
- {2, arcs_53_1},
- {2, arcs_53_2},
+static state states_54[3] = {
+ {1, arcs_54_0},
+ {2, arcs_54_1},
+ {2, arcs_54_2},
};
-static arc arcs_54_0[3] = {
+static arc arcs_55_0[3] = {
{70, 1},
{21, 2},
{14, 3},
};
-static arc arcs_54_1[1] = {
+static arc arcs_55_1[1] = {
{70, 4},
};
-static arc arcs_54_2[2] = {
+static arc arcs_55_2[2] = {
{14, 3},
{0, 2},
};
-static arc arcs_54_3[3] = {
+static arc arcs_55_3[3] = {
{21, 5},
- {142, 6},
+ {144, 6},
{0, 3},
};
-static arc arcs_54_4[1] = {
+static arc arcs_55_4[1] = {
{70, 6},
};
-static arc arcs_54_5[2] = {
- {142, 6},
+static arc arcs_55_5[2] = {
+ {144, 6},
{0, 5},
};
-static arc arcs_54_6[1] = {
+static arc arcs_55_6[1] = {
{0, 6},
};
-static state states_54[7] = {
- {3, arcs_54_0},
- {1, arcs_54_1},
- {2, arcs_54_2},
- {3, arcs_54_3},
- {1, arcs_54_4},
- {2, arcs_54_5},
- {1, arcs_54_6},
-};
-static arc arcs_55_0[1] = {
- {14, 1},
-};
-static arc arcs_55_1[2] = {
- {21, 2},
- {0, 1},
-};
-static arc arcs_55_2[1] = {
- {0, 2},
-};
-static state states_55[3] = {
- {1, arcs_55_0},
- {2, arcs_55_1},
- {1, arcs_55_2},
+static state states_55[7] = {
+ {3, arcs_55_0},
+ {1, arcs_55_1},
+ {2, arcs_55_2},
+ {3, arcs_55_3},
+ {1, arcs_55_4},
+ {2, arcs_55_5},
+ {1, arcs_55_6},
};
static arc arcs_56_0[1] = {
- {73, 1},
+ {14, 1},
};
static arc arcs_56_1[2] = {
- {22, 2},
+ {21, 2},
{0, 1},
};
-static arc arcs_56_2[2] = {
- {73, 1},
+static arc arcs_56_2[1] = {
{0, 2},
};
static state states_56[3] = {
{1, arcs_56_0},
{2, arcs_56_1},
- {2, arcs_56_2},
+ {1, arcs_56_2},
};
static arc arcs_57_0[1] = {
- {21, 1},
+ {73, 1},
};
static arc arcs_57_1[2] = {
{22, 2},
{0, 1},
};
static arc arcs_57_2[2] = {
- {21, 1},
+ {73, 1},
{0, 2},
};
static state states_57[3] = {
@@ -1258,29 +1268,21 @@ static arc arcs_58_1[2] = {
{22, 2},
{0, 1},
};
-static arc arcs_58_2[1] = {
- {21, 3},
-};
-static arc arcs_58_3[2] = {
- {22, 4},
- {0, 3},
-};
-static arc arcs_58_4[2] = {
- {21, 3},
- {0, 4},
+static arc arcs_58_2[2] = {
+ {21, 1},
+ {0, 2},
};
-static state states_58[5] = {
+static state states_58[3] = {
{1, arcs_58_0},
{2, arcs_58_1},
- {1, arcs_58_2},
- {2, arcs_58_3},
- {2, arcs_58_4},
+ {2, arcs_58_2},
};
static arc arcs_59_0[1] = {
{21, 1},
};
-static arc arcs_59_1[1] = {
- {14, 2},
+static arc arcs_59_1[2] = {
+ {22, 2},
+ {0, 1},
};
static arc arcs_59_2[1] = {
{21, 3},
@@ -1290,328 +1292,423 @@ static arc arcs_59_3[2] = {
{0, 3},
};
static arc arcs_59_4[2] = {
- {21, 1},
+ {21, 3},
{0, 4},
};
static state states_59[5] = {
{1, arcs_59_0},
- {1, arcs_59_1},
+ {2, arcs_59_1},
{1, arcs_59_2},
{2, arcs_59_3},
{2, arcs_59_4},
};
static arc arcs_60_0[1] = {
- {144, 1},
+ {21, 1},
};
static arc arcs_60_1[1] = {
+ {14, 2},
+};
+static arc arcs_60_2[1] = {
+ {21, 3},
+};
+static arc arcs_60_3[2] = {
+ {22, 4},
+ {0, 3},
+};
+static arc arcs_60_4[2] = {
+ {21, 1},
+ {0, 4},
+};
+static state states_60[5] = {
+ {1, arcs_60_0},
+ {1, arcs_60_1},
+ {1, arcs_60_2},
+ {2, arcs_60_3},
+ {2, arcs_60_4},
+};
+static arc arcs_61_0[1] = {
+ {146, 1},
+};
+static arc arcs_61_1[1] = {
{12, 2},
};
-static arc arcs_60_2[2] = {
+static arc arcs_61_2[2] = {
{16, 3},
{14, 4},
};
-static arc arcs_60_3[1] = {
+static arc arcs_61_3[1] = {
{9, 5},
};
-static arc arcs_60_4[1] = {
+static arc arcs_61_4[1] = {
{15, 6},
};
-static arc arcs_60_5[1] = {
+static arc arcs_61_5[1] = {
{18, 7},
};
-static arc arcs_60_6[1] = {
+static arc arcs_61_6[1] = {
{0, 6},
};
-static arc arcs_60_7[1] = {
+static arc arcs_61_7[1] = {
{14, 4},
};
-static state states_60[8] = {
- {1, arcs_60_0},
- {1, arcs_60_1},
- {2, arcs_60_2},
- {1, arcs_60_3},
- {1, arcs_60_4},
- {1, arcs_60_5},
- {1, arcs_60_6},
- {1, arcs_60_7},
-};
-static arc arcs_61_0[3] = {
- {145, 1},
+static state states_61[8] = {
+ {1, arcs_61_0},
+ {1, arcs_61_1},
+ {2, arcs_61_2},
+ {1, arcs_61_3},
+ {1, arcs_61_4},
+ {1, arcs_61_5},
+ {1, arcs_61_6},
+ {1, arcs_61_7},
+};
+static arc arcs_62_0[3] = {
+ {147, 1},
{23, 2},
{24, 3},
};
-static arc arcs_61_1[2] = {
+static arc arcs_62_1[2] = {
{22, 4},
{0, 1},
};
-static arc arcs_61_2[1] = {
+static arc arcs_62_2[1] = {
{21, 5},
};
-static arc arcs_61_3[1] = {
+static arc arcs_62_3[1] = {
{21, 6},
};
-static arc arcs_61_4[4] = {
- {145, 1},
+static arc arcs_62_4[4] = {
+ {147, 1},
{23, 2},
{24, 3},
{0, 4},
};
-static arc arcs_61_5[2] = {
+static arc arcs_62_5[2] = {
{22, 7},
{0, 5},
};
-static arc arcs_61_6[1] = {
+static arc arcs_62_6[1] = {
{0, 6},
};
-static arc arcs_61_7[1] = {
+static arc arcs_62_7[1] = {
{24, 3},
};
-static state states_61[8] = {
- {3, arcs_61_0},
- {2, arcs_61_1},
- {1, arcs_61_2},
- {1, arcs_61_3},
- {4, arcs_61_4},
- {2, arcs_61_5},
- {1, arcs_61_6},
- {1, arcs_61_7},
+static state states_62[8] = {
+ {3, arcs_62_0},
+ {2, arcs_62_1},
+ {1, arcs_62_2},
+ {1, arcs_62_3},
+ {4, arcs_62_4},
+ {2, arcs_62_5},
+ {1, arcs_62_6},
+ {1, arcs_62_7},
};
-static arc arcs_62_0[1] = {
+static arc arcs_63_0[1] = {
{21, 1},
};
-static arc arcs_62_1[2] = {
+static arc arcs_63_1[3] = {
{20, 2},
+ {139, 3},
{0, 1},
};
-static arc arcs_62_2[1] = {
- {21, 3},
+static arc arcs_63_2[1] = {
+ {21, 4},
};
-static arc arcs_62_3[1] = {
+static arc arcs_63_3[1] = {
{0, 3},
};
-static state states_62[4] = {
- {1, arcs_62_0},
- {2, arcs_62_1},
- {1, arcs_62_2},
- {1, arcs_62_3},
+static arc arcs_63_4[2] = {
+ {139, 3},
+ {0, 4},
};
-static arc arcs_63_0[2] = {
- {137, 1},
- {147, 1},
+static state states_63[5] = {
+ {1, arcs_63_0},
+ {3, arcs_63_1},
+ {1, arcs_63_2},
+ {1, arcs_63_3},
+ {2, arcs_63_4},
+};
+static arc arcs_64_0[2] = {
+ {138, 1},
+ {149, 1},
};
-static arc arcs_63_1[1] = {
+static arc arcs_64_1[1] = {
{0, 1},
};
-static state states_63[2] = {
- {2, arcs_63_0},
- {1, arcs_63_1},
+static state states_64[2] = {
+ {2, arcs_64_0},
+ {1, arcs_64_1},
};
-static arc arcs_64_0[1] = {
+static arc arcs_65_0[1] = {
{85, 1},
};
-static arc arcs_64_1[1] = {
+static arc arcs_65_1[1] = {
{53, 2},
};
-static arc arcs_64_2[1] = {
+static arc arcs_65_2[1] = {
{74, 3},
};
-static arc arcs_64_3[1] = {
- {143, 4},
+static arc arcs_65_3[1] = {
+ {145, 4},
};
-static arc arcs_64_4[2] = {
- {146, 5},
+static arc arcs_65_4[2] = {
+ {148, 5},
{0, 4},
};
-static arc arcs_64_5[1] = {
+static arc arcs_65_5[1] = {
{0, 5},
};
-static state states_64[6] = {
- {1, arcs_64_0},
- {1, arcs_64_1},
- {1, arcs_64_2},
- {1, arcs_64_3},
- {2, arcs_64_4},
- {1, arcs_64_5},
+static state states_65[6] = {
+ {1, arcs_65_0},
+ {1, arcs_65_1},
+ {1, arcs_65_2},
+ {1, arcs_65_3},
+ {2, arcs_65_4},
+ {1, arcs_65_5},
};
-static arc arcs_65_0[1] = {
+static arc arcs_66_0[1] = {
{81, 1},
};
-static arc arcs_65_1[1] = {
+static arc arcs_66_1[1] = {
{21, 2},
};
-static arc arcs_65_2[2] = {
- {146, 3},
+static arc arcs_66_2[2] = {
+ {148, 3},
{0, 2},
};
-static arc arcs_65_3[1] = {
+static arc arcs_66_3[1] = {
{0, 3},
};
-static state states_65[4] = {
- {1, arcs_65_0},
- {1, arcs_65_1},
- {2, arcs_65_2},
- {1, arcs_65_3},
+static state states_66[4] = {
+ {1, arcs_66_0},
+ {1, arcs_66_1},
+ {2, arcs_66_2},
+ {1, arcs_66_3},
};
-static arc arcs_66_0[1] = {
+static arc arcs_67_0[2] = {
+ {139, 1},
+ {151, 1},
+};
+static arc arcs_67_1[1] = {
+ {0, 1},
+};
+static state states_67[2] = {
+ {2, arcs_67_0},
+ {1, arcs_67_1},
+};
+static arc arcs_68_0[1] = {
+ {85, 1},
+};
+static arc arcs_68_1[1] = {
+ {53, 2},
+};
+static arc arcs_68_2[1] = {
+ {74, 3},
+};
+static arc arcs_68_3[1] = {
+ {21, 4},
+};
+static arc arcs_68_4[2] = {
+ {150, 5},
+ {0, 4},
+};
+static arc arcs_68_5[1] = {
+ {0, 5},
+};
+static state states_68[6] = {
+ {1, arcs_68_0},
+ {1, arcs_68_1},
+ {1, arcs_68_2},
+ {1, arcs_68_3},
+ {2, arcs_68_4},
+ {1, arcs_68_5},
+};
+static arc arcs_69_0[1] = {
+ {81, 1},
+};
+static arc arcs_69_1[1] = {
+ {21, 2},
+};
+static arc arcs_69_2[2] = {
+ {150, 3},
+ {0, 2},
+};
+static arc arcs_69_3[1] = {
+ {0, 3},
+};
+static state states_69[4] = {
+ {1, arcs_69_0},
+ {1, arcs_69_1},
+ {2, arcs_69_2},
+ {1, arcs_69_3},
+};
+static arc arcs_70_0[1] = {
{21, 1},
};
-static arc arcs_66_1[2] = {
+static arc arcs_70_1[2] = {
{22, 0},
{0, 1},
};
-static state states_66[2] = {
- {1, arcs_66_0},
- {2, arcs_66_1},
+static state states_70[2] = {
+ {1, arcs_70_0},
+ {2, arcs_70_1},
};
-static arc arcs_67_0[1] = {
+static arc arcs_71_0[1] = {
{12, 1},
};
-static arc arcs_67_1[1] = {
+static arc arcs_71_1[1] = {
{0, 1},
};
-static state states_67[2] = {
- {1, arcs_67_0},
- {1, arcs_67_1},
+static state states_71[2] = {
+ {1, arcs_71_0},
+ {1, arcs_71_1},
};
-static dfa dfas[68] = {
+static dfa dfas[72] = {
{256, "single_input", 0, 3, states_0,
- "\004\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\244\005\001"},
+ "\004\030\001\000\000\000\124\360\213\011\162\000\002\000\140\010\111\023\004\000"},
{257, "file_input", 0, 2, states_1,
- "\204\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\244\005\001"},
+ "\204\030\001\000\000\000\124\360\213\011\162\000\002\000\140\010\111\023\004\000"},
{258, "eval_input", 0, 3, states_2,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
{259, "funcdef", 0, 6, states_3,
- "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{260, "parameters", 0, 4, states_4,
- "\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{261, "varargslist", 0, 10, states_5,
- "\000\020\201\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\201\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{262, "fpdef", 0, 4, states_6,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{263, "fplist", 0, 3, states_7,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{264, "stmt", 0, 2, states_8,
- "\000\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\244\005\001"},
+ "\000\030\001\000\000\000\124\360\213\011\162\000\002\000\140\010\111\023\004\000"},
{265, "simple_stmt", 0, 4, states_9,
- "\000\020\001\000\000\000\124\360\213\011\000\000\002\000\140\210\244\005\000"},
+ "\000\020\001\000\000\000\124\360\213\011\000\000\002\000\140\010\111\023\000\000"},
{266, "small_stmt", 0, 2, states_10,
- "\000\020\001\000\000\000\124\360\213\011\000\000\002\000\140\210\244\005\000"},
+ "\000\020\001\000\000\000\124\360\213\011\000\000\002\000\140\010\111\023\000\000"},
{267, "expr_stmt", 0, 6, states_11,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
{268, "augassign", 0, 2, states_12,
- "\000\000\000\000\300\377\003\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\300\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{269, "print_stmt", 0, 9, states_13,
- "\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{270, "del_stmt", 0, 3, states_14,
- "\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{271, "pass_stmt", 0, 2, states_15,
- "\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{272, "flow_stmt", 0, 2, states_16,
- "\000\000\000\000\000\000\000\360\001\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\360\001\000\000\000\000\000\000\000\000\000\000\000"},
{273, "break_stmt", 0, 2, states_17,
- "\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"},
{274, "continue_stmt", 0, 2, states_18,
- "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000"},
{275, "return_stmt", 0, 3, states_19,
- "\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"},
{276, "yield_stmt", 0, 3, states_20,
- "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000"},
{277, "raise_stmt", 0, 7, states_21,
- "\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"},
{278, "import_stmt", 0, 9, states_22,
- "\000\000\000\000\000\000\000\000\012\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\012\000\000\000\000\000\000\000\000\000\000\000"},
{279, "import_as_name", 0, 4, states_23,
- "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{280, "dotted_as_name", 0, 4, states_24,
- "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{281, "dotted_name", 0, 2, states_25,
- "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{282, "global_stmt", 0, 3, states_26,
- "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000"},
{283, "exec_stmt", 0, 7, states_27,
- "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000"},
{284, "assert_stmt", 0, 5, states_28,
- "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000"},
{285, "compound_stmt", 0, 2, states_29,
- "\000\010\000\000\000\000\000\000\000\000\162\000\000\000\000\000\000\000\001"},
+ "\000\010\000\000\000\000\000\000\000\000\162\000\000\000\000\000\000\000\004\000"},
{286, "if_stmt", 0, 8, states_30,
- "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"},
{287, "while_stmt", 0, 8, states_31,
- "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"},
{288, "for_stmt", 0, 10, states_32,
- "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"},
{289, "try_stmt", 0, 10, states_33,
- "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"},
{290, "except_clause", 0, 5, states_34,
- "\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000"},
{291, "suite", 0, 5, states_35,
- "\004\020\001\000\000\000\124\360\213\011\000\000\002\000\140\210\244\005\000"},
+ "\004\020\001\000\000\000\124\360\213\011\000\000\002\000\140\010\111\023\000\000"},
{292, "test", 0, 4, states_36,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
{293, "and_test", 0, 2, states_37,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\003\000\000"},
{294, "not_test", 0, 3, states_38,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\003\000\000"},
{295, "comparison", 0, 2, states_39,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{296, "comp_op", 0, 4, states_40,
- "\000\000\000\000\000\000\000\000\000\004\000\000\362\017\000\000\000\000\000"},
+ "\000\000\000\000\000\000\000\000\000\004\000\000\362\017\000\000\000\000\000\000"},
{297, "expr", 0, 2, states_41,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{298, "xor_expr", 0, 2, states_42,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{299, "and_expr", 0, 2, states_43,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{300, "shift_expr", 0, 2, states_44,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{301, "arith_expr", 0, 2, states_45,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{302, "term", 0, 2, states_46,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{303, "factor", 0, 3, states_47,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
{304, "power", 0, 4, states_48,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\200\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\111\003\000\000"},
{305, "atom", 0, 11, states_49,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\200\244\001\000"},
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\111\003\000\000"},
{306, "listmaker", 0, 5, states_50,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {307, "lambdef", 0, 5, states_51,
- "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000"},
- {308, "trailer", 0, 7, states_52,
- "\000\000\001\000\000\000\000\000\100\000\000\000\000\000\000\200\000\000\000"},
- {309, "subscriptlist", 0, 3, states_53,
- "\000\120\001\000\000\000\000\000\100\000\000\000\002\000\140\210\244\005\000"},
- {310, "subscript", 0, 7, states_54,
- "\000\120\001\000\000\000\000\000\100\000\000\000\002\000\140\210\244\005\000"},
- {311, "sliceop", 0, 3, states_55,
- "\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
- {312, "exprlist", 0, 3, states_56,
- "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\244\001\000"},
- {313, "testlist", 0, 3, states_57,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {314, "testlist_safe", 0, 5, states_58,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {315, "dictmaker", 0, 5, states_59,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {316, "classdef", 0, 8, states_60,
- "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"},
- {317, "arglist", 0, 8, states_61,
- "\000\020\201\001\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {318, "argument", 0, 4, states_62,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {319, "list_iter", 0, 2, states_63,
- "\000\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000"},
- {320, "list_for", 0, 6, states_64,
- "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
- {321, "list_if", 0, 4, states_65,
- "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000"},
- {322, "testlist1", 0, 2, states_66,
- "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\244\005\000"},
- {323, "encoding_decl", 0, 2, states_67,
- "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
-};
-static label labels[149] = {
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {307, "testlist_gexp", 0, 5, states_51,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {308, "lambdef", 0, 5, states_52,
+ "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000"},
+ {309, "trailer", 0, 7, states_53,
+ "\000\000\001\000\000\000\000\000\100\000\000\000\000\000\000\000\001\000\000\000"},
+ {310, "subscriptlist", 0, 3, states_54,
+ "\000\120\001\000\000\000\000\000\100\000\000\000\002\000\140\010\111\023\000\000"},
+ {311, "subscript", 0, 7, states_55,
+ "\000\120\001\000\000\000\000\000\100\000\000\000\002\000\140\010\111\023\000\000"},
+ {312, "sliceop", 0, 3, states_56,
+ "\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+ {313, "exprlist", 0, 3, states_57,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\010\111\003\000\000"},
+ {314, "testlist", 0, 3, states_58,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {315, "testlist_safe", 0, 5, states_59,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {316, "dictmaker", 0, 5, states_60,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {317, "classdef", 0, 8, states_61,
+ "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000"},
+ {318, "arglist", 0, 8, states_62,
+ "\000\020\201\001\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {319, "argument", 0, 5, states_63,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {320, "list_iter", 0, 2, states_64,
+ "\000\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000"},
+ {321, "list_for", 0, 6, states_65,
+ "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"},
+ {322, "list_if", 0, 4, states_66,
+ "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"},
+ {323, "gen_iter", 0, 2, states_67,
+ "\000\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000"},
+ {324, "gen_for", 0, 6, states_68,
+ "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"},
+ {325, "gen_if", 0, 4, states_69,
+ "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"},
+ {326, "testlist1", 0, 2, states_70,
+ "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\010\111\023\000\000"},
+ {327, "encoding_decl", 0, 2, states_71,
+ "\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
+};
+static label labels[153] = {
{0, "EMPTY"},
{256, 0},
{4, 0},
@@ -1621,7 +1718,7 @@ static label labels[149] = {
{264, 0},
{0, 0},
{258, 0},
- {313, 0},
+ {314, 0},
{259, 0},
{1, "def"},
{1, 0},
@@ -1665,7 +1762,7 @@ static label labels[149] = {
{1, "print"},
{35, 0},
{1, "del"},
- {312, 0},
+ {313, 0},
{1, "pass"},
{273, 0},
{274, 0},
@@ -1692,7 +1789,7 @@ static label labels[149] = {
{287, 0},
{288, 0},
{289, 0},
- {316, 0},
+ {317, 0},
{1, "if"},
{1, "elif"},
{1, "else"},
@@ -1706,7 +1803,7 @@ static label labels[149] = {
{6, 0},
{293, 0},
{1, "or"},
- {307, 0},
+ {308, 0},
{294, 0},
{1, "and"},
{1, "not"},
@@ -1738,33 +1835,37 @@ static label labels[149] = {
{32, 0},
{304, 0},
{305, 0},
- {308, 0},
+ {309, 0},
+ {307, 0},
{9, 0},
{306, 0},
{10, 0},
{26, 0},
- {315, 0},
+ {316, 0},
{27, 0},
{25, 0},
- {322, 0},
+ {326, 0},
{2, 0},
{3, 0},
- {320, 0},
+ {321, 0},
+ {324, 0},
{1, "lambda"},
- {317, 0},
- {309, 0},
+ {318, 0},
{310, 0},
{311, 0},
- {314, 0},
+ {312, 0},
+ {315, 0},
{1, "class"},
- {318, 0},
{319, 0},
- {321, 0},
+ {320, 0},
+ {322, 0},
{323, 0},
+ {325, 0},
+ {327, 0},
};
grammar _PyParser_Grammar = {
- 68,
+ 72,
dfas,
- {149, labels},
+ {153, labels},
256
};
diff --git a/Python/symtable.c b/Python/symtable.c
index f86fd2fbc8..5ca204164a 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -66,6 +66,8 @@ PySymtableEntry_New(struct symtable *st, char *name, int type, int lineno)
switch (type) {
case funcdef:
case lambdef:
+ case testlist_gexp: /* generator expression */
+ case argument: /* generator expression */
ste->ste_type = TYPE_FUNCTION;
break;
case classdef:
diff --git a/Tools/compiler/ast.txt b/Tools/compiler/ast.txt
index 3f8953bea8..6a990d4cc5 100644
--- a/Tools/compiler/ast.txt
+++ b/Tools/compiler/ast.txt
@@ -38,6 +38,10 @@ AssAttr: expr, attrname*, flags*
ListComp: expr, quals!
ListCompFor: assign, list, ifs!
ListCompIf: test
+GenExpr: code
+GenExprInner: expr, quals!
+GenExprFor: assign, iter, ifs!
+GenExprIf: test
List: nodes!
Dict: items!
Not: expr
@@ -85,3 +89,10 @@ init(Lambda):
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
+
+init(GenExpr):
+ self.argnames = ['[outmost-iterable]']
+ self.varargs = self.kwargs = None
+
+init(GenExprFor):
+ self.is_outmost = False
View Lorry Controller Status