summaryrefslogtreecommitdiff
path: root/Lib/tokenize.py
diff options
context:
space:
mode:
authorTim Peters <tim.peters@gmail.com>2002-05-23 15:15:30 +0000
committerTim Peters <tim.peters@gmail.com>2002-05-23 15:15:30 +0000
commit8ac1495a6a1d18111a626cec0c7f2eb67df3edb3 (patch)
tree2d91993770d3a5b3f3668857983d9bf75276b14f /Lib/tokenize.py
parentf655328483b2e237cc2f71c1c308eceb2f30f6fd (diff)
downloadcpython-git-8ac1495a6a1d18111a626cec0c7f2eb67df3edb3.tar.gz
Whitespace normalization.
Diffstat (limited to 'Lib/tokenize.py')
-rw-r--r--Lib/tokenize.py10
1 files changed, 5 insertions, 5 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 42aafe4723..22f28c4426 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -124,14 +124,14 @@ def tokenize(readline, tokeneater=printtoken):
"""
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
-
+
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
- Each call to the function should return one line of input as a string.
+ Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
- tuples generated by generate_tokens().
+ tuples generated by generate_tokens().
"""
try:
tokenize_loop(readline, tokeneater)
@@ -149,13 +149,13 @@ def generate_tokens(readline):
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string.
-
+
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
- logical line; continuation lines are included.
+ logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
namechars, numchars = string.ascii_letters + '_', '0123456789'