summaryrefslogtreecommitdiff
path: root/babel/messages/jslexer.py
diff options
context:
space:
mode:
authorArmin Ronacher <armin.ronacher@active-4.com>2008-06-14 19:00:35 +0000
committerArmin Ronacher <armin.ronacher@active-4.com>2008-06-14 19:00:35 +0000
commit2f4ca2051945343f0ec217d9f83e0f96d7f3eb0d (patch)
tree4b9e5ff6d0521c96086fa7581c70511879160cd8 /babel/messages/jslexer.py
parent4626f01a103487f3118e362ccf48e74ca315101c (diff)
downloadbabel-2f4ca2051945343f0ec217d9f83e0f96d7f3eb0d.tar.gz
added some newlines to extract and jslexer to stay consistent with the rest of the sourcecode.
Diffstat (limited to 'babel/messages/jslexer.py')
-rw-r--r--babel/messages/jslexer.py4
1 files changed, 4 insertions, 0 deletions
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index a5f4436..9c666f0 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -58,6 +58,7 @@ uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
class TokenError(ValueError):
"""Raised if the tokenizer stumbled upon invalid tokens."""
+
class Token(tuple):
"""Represents a token as returned by `tokenize`."""
__slots__ = ()
@@ -69,6 +70,7 @@ class Token(tuple):
value = property(itemgetter(1))
lineno = property(itemgetter(2))
+
def indicates_division(token):
"""A helper function that helps the tokenizer to decide if the current
token may be followed by a division operator.
@@ -77,6 +79,7 @@ def indicates_division(token):
return token.value in (')', ']', '}', '++', '--')
return token.type in ('name', 'number', 'string', 'regexp')
+
def unquote_string(string):
"""Unquote a string with JavaScript rules. The string has to start with
string delimiters (``'`` or ``"``.)
@@ -134,6 +137,7 @@ def unquote_string(string):
return u''.join(result)
+
def tokenize(source):
"""Tokenize a JavaScript source.