webkitgtk-2.16.5HEADwebkitgtk-2.16.5master

1 files changed, 879 insertions, 0 deletions

diff --git a/Source/WebCore/css/parser/CSSTokenizer.cpp b/Source/WebCore/css/parser/CSSTokenizer.cpp
new file mode 100644
index 000000000..1f1a23e4d
--- /dev/null
+++ b/Source/WebCore/css/parser/CSSTokenizer.cpp
@@ -0,0 +1,879 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Copyright (C) 2016 Apple Inc. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "config.h"
+#include "CSSTokenizer.h"
+
+#include "CSSParserIdioms.h"
+#include "CSSParserObserverWrapper.h"
+#include "CSSParserTokenRange.h"
+#include "CSSTokenizerInputStream.h"
+#include "HTMLParserIdioms.h"
+#include <wtf/text/StringBuilder.h>
+#include <wtf/unicode/CharacterNames.h>
+
+namespace WebCore {
+
+CSSTokenizer::CSSTokenizer(const String& string)
+    : m_input(string)
+{
+    // According to the spec, we should perform preprocessing here.
+    // See: http://dev.w3.org/csswg/css-syntax/#input-preprocessing
+    //
+    // However, we can skip this step since:
+    // * We're using HTML spaces (which accept \r and \f as a valid white space)
+    // * Do not count white spaces
+    // * CSSTokenizerInputStream::nextInputChar() replaces NULLs for replacement characters
+
+    if (string.isEmpty())
+        return;
+
+    // To avoid resizing we err on the side of reserving too much space.
+    // Most strings we tokenize have about 3.5 to 5 characters per token.
+    m_tokens.reserveInitialCapacity(string.length() / 3);
+
+    while (true) {
+        CSSParserToken token = nextToken();
+        if (token.type() == CommentToken)
+            continue;
+        if (token.type() == EOFToken)
+            return;
+        m_tokens.append(token);
+    }
+}
+
+CSSTokenizer::CSSTokenizer(const String& string, CSSParserObserverWrapper& wrapper)
+    : m_input(string)
+{
+    if (string.isEmpty())
+        return;
+
+    unsigned offset = 0;
+    while (true) {
+        CSSParserToken token = nextToken();
+        if (token.type() == EOFToken)
+            break;
+        if (token.type() == CommentToken)
+            wrapper.addComment(offset, m_input.offset(), m_tokens.size());
+        else {
+            m_tokens.append(token);
+            wrapper.addToken(offset);
+        }
+        offset = m_input.offset();
+    }
+
+    wrapper.addToken(offset);
+    wrapper.finalizeConstruction(m_tokens.begin());
+}
+
+CSSParserTokenRange CSSTokenizer::tokenRange() const
+{
+    return m_tokens;
+}
+
+unsigned CSSTokenizer::tokenCount()
+{
+    return m_tokens.size();
+}
+
+static bool isNewLine(UChar cc)
+{
+    // We check \r and \f here, since we have no preprocessing stage
+    return (cc == '\r' || cc == '\n' || cc == '\f');
+}
+
+// http://dev.w3.org/csswg/css-syntax/#check-if-two-code-points-are-a-valid-escape
+static bool twoCharsAreValidEscape(UChar first, UChar second)
+{
+    return first == '\\' && !isNewLine(second);
+}
+
+void CSSTokenizer::reconsume(UChar c)
+{
+    m_input.pushBack(c);
+}
+
+UChar CSSTokenizer::consume()
+{
+    UChar current = m_input.nextInputChar();
+    m_input.advance();
+    return current;
+}
+
+CSSParserToken CSSTokenizer::whiteSpace(UChar /*cc*/)
+{
+    m_input.advanceUntilNonWhitespace();
+    return CSSParserToken(WhitespaceToken);
+}
+
+CSSParserToken CSSTokenizer::blockStart(CSSParserTokenType type)
+{
+    m_blockStack.append(type);
+    return CSSParserToken(type, CSSParserToken::BlockStart);
+}
+
+CSSParserToken CSSTokenizer::blockStart(CSSParserTokenType blockType, CSSParserTokenType type, StringView name)
+{
+    m_blockStack.append(blockType);
+    return CSSParserToken(type, name, CSSParserToken::BlockStart);
+}
+
+CSSParserToken CSSTokenizer::blockEnd(CSSParserTokenType type, CSSParserTokenType startType)
+{
+    if (!m_blockStack.isEmpty() && m_blockStack.last() == startType) {
+        m_blockStack.removeLast();
+        return CSSParserToken(type, CSSParserToken::BlockEnd);
+    }
+    return CSSParserToken(type);
+}
+
+CSSParserToken CSSTokenizer::leftParenthesis(UChar /*cc*/)
+{
+    return blockStart(LeftParenthesisToken);
+}
+
+CSSParserToken CSSTokenizer::rightParenthesis(UChar /*cc*/)
+{
+    return blockEnd(RightParenthesisToken, LeftParenthesisToken);
+}
+
+CSSParserToken CSSTokenizer::leftBracket(UChar /*cc*/)
+{
+    return blockStart(LeftBracketToken);
+}
+
+CSSParserToken CSSTokenizer::rightBracket(UChar /*cc*/)
+{
+    return blockEnd(RightBracketToken, LeftBracketToken);
+}
+
+CSSParserToken CSSTokenizer::leftBrace(UChar /*cc*/)
+{
+    return blockStart(LeftBraceToken);
+}
+
+CSSParserToken CSSTokenizer::rightBrace(UChar /*cc*/)
+{
+    return blockEnd(RightBraceToken, LeftBraceToken);
+}
+
+CSSParserToken CSSTokenizer::plusOrFullStop(UChar cc)
+{
+    if (nextCharsAreNumber(cc)) {
+        reconsume(cc);
+        return consumeNumericToken();
+    }
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+CSSParserToken CSSTokenizer::asterisk(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '*');
+    if (consumeIfNext('='))
+        return CSSParserToken(SubstringMatchToken);
+    return CSSParserToken(DelimiterToken, '*');
+}
+
+CSSParserToken CSSTokenizer::lessThan(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '<');
+    if (m_input.peekWithoutReplacement(0) == '!'
+        && m_input.peekWithoutReplacement(1) == '-'
+        && m_input.peekWithoutReplacement(2) == '-') {
+        m_input.advance(3);
+        return CSSParserToken(CDOToken);
+    }
+    return CSSParserToken(DelimiterToken, '<');
+}
+
+CSSParserToken CSSTokenizer::comma(UChar /*cc*/)
+{
+    return CSSParserToken(CommaToken);
+}
+
+CSSParserToken CSSTokenizer::hyphenMinus(UChar cc)
+{
+    if (nextCharsAreNumber(cc)) {
+        reconsume(cc);
+        return consumeNumericToken();
+    }
+    if (m_input.peekWithoutReplacement(0) == '-'
+        && m_input.peekWithoutReplacement(1) == '>') {
+        m_input.advance(2);
+        return CSSParserToken(CDCToken);
+    }
+    if (nextCharsAreIdentifier(cc)) {
+        reconsume(cc);
+        return consumeIdentLikeToken();
+    }
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+CSSParserToken CSSTokenizer::solidus(UChar cc)
+{
+    if (consumeIfNext('*')) {
+        // These get ignored, but we need a value to return.
+        consumeUntilCommentEndFound();
+        return CSSParserToken(CommentToken);
+    }
+
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+CSSParserToken CSSTokenizer::colon(UChar /*cc*/)
+{
+    return CSSParserToken(ColonToken);
+}
+
+CSSParserToken CSSTokenizer::semiColon(UChar /*cc*/)
+{
+    return CSSParserToken(SemicolonToken);
+}
+
+CSSParserToken CSSTokenizer::hash(UChar cc)
+{
+    UChar nextChar = m_input.peekWithoutReplacement(0);
+    if (isNameCodePoint(nextChar) || twoCharsAreValidEscape(nextChar, m_input.peekWithoutReplacement(1))) {
+        HashTokenType type = nextCharsAreIdentifier() ? HashTokenId : HashTokenUnrestricted;
+        return CSSParserToken(type, consumeName());
+    }
+
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+CSSParserToken CSSTokenizer::circumflexAccent(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '^');
+    if (consumeIfNext('='))
+        return CSSParserToken(PrefixMatchToken);
+    return CSSParserToken(DelimiterToken, '^');
+}
+
+CSSParserToken CSSTokenizer::dollarSign(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '$');
+    if (consumeIfNext('='))
+        return CSSParserToken(SuffixMatchToken);
+    return CSSParserToken(DelimiterToken, '$');
+}
+
+CSSParserToken CSSTokenizer::verticalLine(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '|');
+    if (consumeIfNext('='))
+        return CSSParserToken(DashMatchToken);
+    if (consumeIfNext('|'))
+        return CSSParserToken(ColumnToken);
+    return CSSParserToken(DelimiterToken, '|');
+}
+
+CSSParserToken CSSTokenizer::tilde(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '~');
+    if (consumeIfNext('='))
+        return CSSParserToken(IncludeMatchToken);
+    return CSSParserToken(DelimiterToken, '~');
+}
+
+CSSParserToken CSSTokenizer::commercialAt(UChar cc)
+{
+    ASSERT_UNUSED(cc, cc == '@');
+    if (nextCharsAreIdentifier())
+        return CSSParserToken(AtKeywordToken, consumeName());
+    return CSSParserToken(DelimiterToken, '@');
+}
+
+CSSParserToken CSSTokenizer::reverseSolidus(UChar cc)
+{
+    if (twoCharsAreValidEscape(cc, m_input.peekWithoutReplacement(0))) {
+        reconsume(cc);
+        return consumeIdentLikeToken();
+    }
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+CSSParserToken CSSTokenizer::asciiDigit(UChar cc)
+{
+    reconsume(cc);
+    return consumeNumericToken();
+}
+
+CSSParserToken CSSTokenizer::letterU(UChar cc)
+{
+    if (m_input.peekWithoutReplacement(0) == '+'
+        && (isASCIIHexDigit(m_input.peekWithoutReplacement(1))
+            || m_input.peekWithoutReplacement(1) == '?')) {
+        m_input.advance();
+        return consumeUnicodeRange();
+    }
+    reconsume(cc);
+    return consumeIdentLikeToken();
+}
+
+CSSParserToken CSSTokenizer::nameStart(UChar cc)
+{
+    reconsume(cc);
+    return consumeIdentLikeToken();
+}
+
+CSSParserToken CSSTokenizer::stringStart(UChar cc)
+{
+    return consumeStringTokenUntil(cc);
+}
+
+CSSParserToken CSSTokenizer::endOfFile(UChar /*cc*/)
+{
+    return CSSParserToken(EOFToken);
+}
+
+const CSSTokenizer::CodePoint CSSTokenizer::codePoints[128] = {
+    &CSSTokenizer::endOfFile,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    &CSSTokenizer::whiteSpace,
+    &CSSTokenizer::whiteSpace,
+    0,
+    &CSSTokenizer::whiteSpace,
+    &CSSTokenizer::whiteSpace,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    &CSSTokenizer::whiteSpace,
+    0,
+    &CSSTokenizer::stringStart,
+    &CSSTokenizer::hash,
+    &CSSTokenizer::dollarSign,
+    0,
+    0,
+    &CSSTokenizer::stringStart,
+    &CSSTokenizer::leftParenthesis,
+    &CSSTokenizer::rightParenthesis,
+    &CSSTokenizer::asterisk,
+    &CSSTokenizer::plusOrFullStop,
+    &CSSTokenizer::comma,
+    &CSSTokenizer::hyphenMinus,
+    &CSSTokenizer::plusOrFullStop,
+    &CSSTokenizer::solidus,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::asciiDigit,
+    &CSSTokenizer::colon,
+    &CSSTokenizer::semiColon,
+    &CSSTokenizer::lessThan,
+    0,
+    0,
+    0,
+    &CSSTokenizer::commercialAt,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::letterU,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::leftBracket,
+    &CSSTokenizer::reverseSolidus,
+    &CSSTokenizer::rightBracket,
+    &CSSTokenizer::circumflexAccent,
+    &CSSTokenizer::nameStart,
+    0,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::letterU,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::nameStart,
+    &CSSTokenizer::leftBrace,
+    &CSSTokenizer::verticalLine,
+    &CSSTokenizer::rightBrace,
+    &CSSTokenizer::tilde,
+    0,
+};
+#if !ASSERT_WITH_SECURITY_IMPLICATION_DISABLED
+const unsigned codePointsNumber = 128;
+#endif
+
+CSSParserToken CSSTokenizer::nextToken()
+{
+    // Unlike the HTMLTokenizer, the CSS Syntax spec is written
+    // as a stateless, (fixed-size) look-ahead tokenizer.
+    // We could move to the stateful model and instead create
+    // states for all the "next 3 codepoints are X" cases.
+    // State-machine tokenizers are easier to write to handle
+    // incremental tokenization of partial sources.
+    // However, for now we follow the spec exactly.
+    UChar cc = consume();
+    CodePoint codePointFunc = 0;
+
+    if (isASCII(cc)) {
+        ASSERT_WITH_SECURITY_IMPLICATION(cc < codePointsNumber);
+        codePointFunc = codePoints[cc];
+    } else
+        codePointFunc = &CSSTokenizer::nameStart;
+
+    if (codePointFunc)
+        return ((this)->*(codePointFunc))(cc);
+    return CSSParserToken(DelimiterToken, cc);
+}
+
+// This method merges the following spec sections for efficiency
+// http://www.w3.org/TR/css3-syntax/#consume-a-number
+// http://www.w3.org/TR/css3-syntax/#convert-a-string-to-a-number
+CSSParserToken CSSTokenizer::consumeNumber()
+{
+    ASSERT(nextCharsAreNumber());
+
+    NumericValueType type = IntegerValueType;
+    NumericSign sign = NoSign;
+    unsigned numberLength = 0;
+
+    UChar next = m_input.peekWithoutReplacement(0);
+    if (next == '+') {
+        ++numberLength;
+        sign = PlusSign;
+    } else if (next == '-') {
+        ++numberLength;
+        sign = MinusSign;
+    }
+
+    numberLength = m_input.skipWhilePredicate<isASCIIDigit>(numberLength);
+    next = m_input.peekWithoutReplacement(numberLength);
+    if (next == '.' && isASCIIDigit(m_input.peekWithoutReplacement(numberLength + 1))) {
+        type = NumberValueType;
+        numberLength = m_input.skipWhilePredicate<isASCIIDigit>(numberLength + 2);
+        next = m_input.peekWithoutReplacement(numberLength);
+    }
+
+    if (next == 'E' || next == 'e') {
+        next = m_input.peekWithoutReplacement(numberLength + 1);
+        if (isASCIIDigit(next)) {
+            type = NumberValueType;
+            numberLength = m_input.skipWhilePredicate<isASCIIDigit>(numberLength + 1);
+        } else if ((next == '+' || next == '-') && isASCIIDigit(m_input.peekWithoutReplacement(numberLength + 2))) {
+            type = NumberValueType;
+            numberLength = m_input.skipWhilePredicate<isASCIIDigit>(numberLength + 3);
+        }
+    }
+
+    double value = m_input.getDouble(0, numberLength);
+    m_input.advance(numberLength);
+
+    return CSSParserToken(NumberToken, value, type, sign);
+}
+
+// http://www.w3.org/TR/css3-syntax/#consume-a-numeric-token
+CSSParserToken CSSTokenizer::consumeNumericToken()
+{
+    CSSParserToken token = consumeNumber();
+    if (nextCharsAreIdentifier())
+        token.convertToDimensionWithUnit(consumeName());
+    else if (consumeIfNext('%'))
+        token.convertToPercentage();
+    return token;
+}
+
+// http://dev.w3.org/csswg/css-syntax/#consume-ident-like-token
+CSSParserToken CSSTokenizer::consumeIdentLikeToken()
+{
+    StringView name = consumeName();
+    if (consumeIfNext('(')) {
+        if (equalIgnoringASCIICase(name, "url")) {
+            // The spec is slightly different so as to avoid dropping whitespace
+            // tokens, but they wouldn't be used and this is easier.
+            m_input.advanceUntilNonWhitespace();
+            UChar next = m_input.peekWithoutReplacement(0);
+            if (next != '"' && next != '\'')
+                return consumeUrlToken();
+        }
+        return blockStart(LeftParenthesisToken, FunctionToken, name);
+    }
+    return CSSParserToken(IdentToken, name);
+}
+
+// http://dev.w3.org/csswg/css-syntax/#consume-a-string-token
+CSSParserToken CSSTokenizer::consumeStringTokenUntil(UChar endingCodePoint)
+{
+    // Strings without escapes get handled without allocations
+    for (unsigned size = 0; ; size++) {
+        UChar cc = m_input.peekWithoutReplacement(size);
+        if (cc == endingCodePoint) {
+            unsigned startOffset = m_input.offset();
+            m_input.advance(size + 1);
+            return CSSParserToken(StringToken, m_input.rangeAt(startOffset, size));
+        }
+        if (isNewLine(cc)) {
+            m_input.advance(size);
+            return CSSParserToken(BadStringToken);
+        }
+        if (cc == '\0' || cc == '\\')
+            break;
+    }
+
+    StringBuilder output;
+    while (true) {
+        UChar cc = consume();
+        if (cc == endingCodePoint || cc == kEndOfFileMarker)
+            return CSSParserToken(StringToken, registerString(output.toString()));
+        if (isNewLine(cc)) {
+            reconsume(cc);
+            return CSSParserToken(BadStringToken);
+        }
+        if (cc == '\\') {
+            if (m_input.nextInputChar() == kEndOfFileMarker)
+                continue;
+            if (isNewLine(m_input.peekWithoutReplacement(0)))
+                consumeSingleWhitespaceIfNext(); // This handles \r\n for us
+            else
+                output.append(consumeEscape());
+        } else
+            output.append(cc);
+    }
+}
+
+CSSParserToken CSSTokenizer::consumeUnicodeRange()
+{
+    ASSERT(isASCIIHexDigit(m_input.peekWithoutReplacement(0)) || m_input.peekWithoutReplacement(0) == '?');
+    int lengthRemaining = 6;
+    UChar32 start = 0;
+
+    while (lengthRemaining && isASCIIHexDigit(m_input.peekWithoutReplacement(0))) {
+        start = start * 16 + toASCIIHexValue(consume());
+        --lengthRemaining;
+    }
+
+    UChar32 end = start;
+    if (lengthRemaining && consumeIfNext('?')) {
+        do {
+            start *= 16;
+            end = end * 16 + 0xF;
+            --lengthRemaining;
+        } while (lengthRemaining && consumeIfNext('?'));
+    } else if (m_input.peekWithoutReplacement(0) == '-' && isASCIIHexDigit(m_input.peekWithoutReplacement(1))) {
+        m_input.advance();
+        lengthRemaining = 6;
+        end = 0;
+        do {
+            end = end * 16 + toASCIIHexValue(consume());
+            --lengthRemaining;
+        } while (lengthRemaining && isASCIIHexDigit(m_input.peekWithoutReplacement(0)));
+    }
+
+    return CSSParserToken(UnicodeRangeToken, start, end);
+}
+
+// http://dev.w3.org/csswg/css-syntax/#non-printable-code-point
+static bool isNonPrintableCodePoint(UChar cc)
+{
+    return cc <= '\x8' || cc == '\xb' || (cc >= '\xe' && cc <= '\x1f') || cc == '\x7f';
+}
+
+// http://dev.w3.org/csswg/css-syntax/#consume-url-token
+CSSParserToken CSSTokenizer::consumeUrlToken()
+{
+    m_input.advanceUntilNonWhitespace();
+
+    // URL tokens without escapes get handled without allocations
+    for (unsigned size = 0; ; size++) {
+        UChar cc = m_input.peekWithoutReplacement(size);
+        if (cc == ')') {
+            unsigned startOffset = m_input.offset();
+            m_input.advance(size + 1);
+            return CSSParserToken(UrlToken, m_input.rangeAt(startOffset, size));
+        }
+        if (cc <= ' ' || cc == '\\' || cc == '"' || cc == '\'' || cc == '(' || cc == '\x7f')
+            break;
+    }
+
+    StringBuilder result;
+    while (true) {
+        UChar cc = consume();
+        if (cc == ')' || cc == kEndOfFileMarker)
+            return CSSParserToken(UrlToken, registerString(result.toString()));
+
+        if (isHTMLSpace(cc)) {
+            m_input.advanceUntilNonWhitespace();
+            if (consumeIfNext(')') || m_input.nextInputChar() == kEndOfFileMarker)
+                return CSSParserToken(UrlToken, registerString(result.toString()));
+            break;
+        }
+
+        if (cc == '"' || cc == '\'' || cc == '(' || isNonPrintableCodePoint(cc))
+            break;
+
+        if (cc == '\\') {
+            if (twoCharsAreValidEscape(cc, m_input.peekWithoutReplacement(0))) {
+                result.append(consumeEscape());
+                continue;
+            }
+            break;
+        }
+
+        result.append(cc);
+    }
+
+    consumeBadUrlRemnants();
+    return CSSParserToken(BadUrlToken);
+}
+
+// http://dev.w3.org/csswg/css-syntax/#consume-the-remnants-of-a-bad-url
+void CSSTokenizer::consumeBadUrlRemnants()
+{
+    while (true) {
+        UChar cc = consume();
+        if (cc == ')' || cc == kEndOfFileMarker)
+            return;
+        if (twoCharsAreValidEscape(cc, m_input.peekWithoutReplacement(0)))
+            consumeEscape();
+    }
+}
+
+void CSSTokenizer::consumeSingleWhitespaceIfNext()
+{
+    // We check for \r\n and HTML spaces since we don't do preprocessing
+    UChar next = m_input.peekWithoutReplacement(0);
+    if (next == '\r' && m_input.peekWithoutReplacement(1) == '\n')
+        m_input.advance(2);
+    else if (isHTMLSpace(next))
+        m_input.advance();
+}
+
+void CSSTokenizer::consumeUntilCommentEndFound()
+{
+    UChar c = consume();
+    while (true) {
+        if (c == kEndOfFileMarker)
+            return;
+        if (c != '*') {
+            c = consume();
+            continue;
+        }
+        c = consume();
+        if (c == '/')
+            return;
+    }
+}
+
+bool CSSTokenizer::consumeIfNext(UChar character)
+{
+    // Since we're not doing replacement we can't tell the difference from
+    // a NUL in the middle and the kEndOfFileMarker, so character must not be
+    // NUL.
+    ASSERT(character);
+    if (m_input.peekWithoutReplacement(0) == character) {
+        m_input.advance();
+        return true;
+    }
+    return false;
+}
+
+// http://www.w3.org/TR/css3-syntax/#consume-a-name
+StringView CSSTokenizer::consumeName()
+{
+    // Names without escapes get handled without allocations
+    for (unsigned size = 0; ; ++size) {
+        UChar cc = m_input.peekWithoutReplacement(size);
+        if (isNameCodePoint(cc))
+            continue;
+        // peekWithoutReplacement will return NUL when we hit the end of the
+        // input. In that case we want to still use the rangeAt() fast path
+        // below.
+        if (cc == '\0' && m_input.offset() + size < m_input.length())
+            break;
+        if (cc == '\\')
+            break;
+        unsigned startOffset = m_input.offset();
+        m_input.advance(size);
+        return m_input.rangeAt(startOffset, size);
+    }
+
+    StringBuilder result;
+    while (true) {
+        UChar cc = consume();
+        if (isNameCodePoint(cc)) {
+            result.append(cc);
+            continue;
+        }
+        if (twoCharsAreValidEscape(cc, m_input.peekWithoutReplacement(0))) {
+            result.append(consumeEscape());
+            continue;
+        }
+        reconsume(cc);
+        return registerString(result.toString());
+    }
+}
+
+// http://dev.w3.org/csswg/css-syntax/#consume-an-escaped-code-point
+UChar32 CSSTokenizer::consumeEscape()
+{
+    UChar cc = consume();
+    ASSERT(!isNewLine(cc));
+    if (isASCIIHexDigit(cc)) {
+        unsigned consumedHexDigits = 1;
+        StringBuilder hexChars;
+        hexChars.append(cc);
+        while (consumedHexDigits < 6 && isASCIIHexDigit(m_input.peekWithoutReplacement(0))) {
+            cc = consume();
+            hexChars.append(cc);
+            consumedHexDigits++;
+        };
+        consumeSingleWhitespaceIfNext();
+        bool ok = false;
+        UChar32 codePoint = hexChars.toString().toUIntStrict(&ok, 16);
+        ASSERT(ok);
+        if (!codePoint || (0xD800 <= codePoint && codePoint <= 0xDFFF) || codePoint > 0x10FFFF)
+            return replacementCharacter;
+        return codePoint;
+    }
+
+    if (cc == kEndOfFileMarker)
+        return replacementCharacter;
+    return cc;
+}
+
+bool CSSTokenizer::nextTwoCharsAreValidEscape()
+{
+    return twoCharsAreValidEscape(m_input.peekWithoutReplacement(0), m_input.peekWithoutReplacement(1));
+}
+
+// http://www.w3.org/TR/css3-syntax/#starts-with-a-number
+bool CSSTokenizer::nextCharsAreNumber(UChar first)
+{
+    UChar second = m_input.peekWithoutReplacement(0);
+    if (isASCIIDigit(first))
+        return true;
+    if (first == '+' || first == '-')
+        return ((isASCIIDigit(second)) || (second == '.' && isASCIIDigit(m_input.peekWithoutReplacement(1))));
+    if (first =='.')
+        return (isASCIIDigit(second));
+    return false;
+}
+
+bool CSSTokenizer::nextCharsAreNumber()
+{
+    UChar first = consume();
+    bool areNumber = nextCharsAreNumber(first);
+    reconsume(first);
+    return areNumber;
+}
+
+// http://dev.w3.org/csswg/css-syntax/#would-start-an-identifier
+bool CSSTokenizer::nextCharsAreIdentifier(UChar first)
+{
+    UChar second = m_input.peekWithoutReplacement(0);
+    if (isNameStartCodePoint(first) || twoCharsAreValidEscape(first, second))
+        return true;
+
+    if (first == '-')
+        return isNameStartCodePoint(second) || second == '-' || nextTwoCharsAreValidEscape();
+
+    return false;
+}
+
+bool CSSTokenizer::nextCharsAreIdentifier()
+{
+    UChar first = consume();
+    bool areIdentifier = nextCharsAreIdentifier(first);
+    reconsume(first);
+    return areIdentifier;
+}
+
+StringView CSSTokenizer::registerString(const String& string)
+{
+    m_stringPool.append(string);
+    return string;
+}
+
+} // namespace WebCore