summaryrefslogtreecommitdiff
path: root/libgo/go/html/token.go
diff options
context:
space:
mode:
Diffstat (limited to 'libgo/go/html/token.go')
-rw-r--r--libgo/go/html/token.go707
1 files changed, 436 insertions, 271 deletions
diff --git a/libgo/go/html/token.go b/libgo/go/html/token.go
index d266b3a300b..2826f95f17f 100644
--- a/libgo/go/html/token.go
+++ b/libgo/go/html/token.go
@@ -9,6 +9,7 @@ import (
"io"
"os"
"strconv"
+ "strings"
)
// A TokenType is the type of a Token.
@@ -100,13 +101,19 @@ func (t Token) String() string {
case SelfClosingTagToken:
return "<" + t.tagString() + "/>"
case CommentToken:
- return "<!--" + EscapeString(t.Data) + "-->"
+ return "<!--" + t.Data + "-->"
case DoctypeToken:
- return "<!DOCTYPE " + EscapeString(t.Data) + ">"
+ return "<!DOCTYPE " + t.Data + ">"
}
return "Invalid(" + strconv.Itoa(int(t.Type)) + ")"
}
+// span is a range of bytes in a Tokenizer's buffer. The start is inclusive,
+// the end is exclusive.
+type span struct {
+ start, end int
+}
+
// A Tokenizer returns a stream of HTML Tokens.
type Tokenizer struct {
// If ReturnComments is set, Next returns comment tokens;
@@ -115,7 +122,7 @@ type Tokenizer struct {
// r is the source of the HTML text.
r io.Reader
- // tt is the TokenType of the most recently read token.
+ // tt is the TokenType of the current token.
tt TokenType
// err is the first error encountered during tokenization. It is possible
// for tt != Error && err != nil to hold: this means that Next returned a
@@ -125,10 +132,26 @@ type Tokenizer struct {
// subsequent Next calls would return an ErrorToken.
// err is never reset. Once it becomes non-nil, it stays non-nil.
err os.Error
- // buf[p0:p1] holds the raw data of the most recent token.
- // buf[p1:] is buffered input that will yield future tokens.
- p0, p1 int
- buf []byte
+ // buf[raw.start:raw.end] holds the raw bytes of the current token.
+ // buf[raw.end:] is buffered input that will yield future tokens.
+ raw span
+ buf []byte
+ // buf[data.start:data.end] holds the raw bytes of the current token's data:
+ // a text token's text, a tag token's tag name, etc.
+ data span
+ // pendingAttr is the attribute key and value currently being tokenized.
+ // When complete, pendingAttr is pushed onto attr. nAttrReturned is
+ // incremented on each call to TagAttr.
+ pendingAttr [2]span
+ attr [][2]span
+ nAttrReturned int
+ // rawTag is the "script" in "</script>" that closes the next token. If
+ // non-empty, the subsequent call to Next will return a raw or RCDATA text
+ // token: one that treats "<p>" as text instead of an element.
+ // rawTag's contents are lower-cased.
+ rawTag string
+ // textIsRaw is whether the current text token's data is not escaped.
+ textIsRaw bool
}
// Error returns the error associated with the most recent ErrorToken token.
@@ -140,33 +163,42 @@ func (z *Tokenizer) Error() os.Error {
return z.err
}
-// Raw returns the unmodified text of the current token. Calling Next, Token,
-// Text, TagName or TagAttr may change the contents of the returned slice.
-func (z *Tokenizer) Raw() []byte {
- return z.buf[z.p0:z.p1]
-}
-
// readByte returns the next byte from the input stream, doing a buffered read
-// from z.r into z.buf if necessary. z.buf[z.p0:z.p1] remains a contiguous byte
+// from z.r into z.buf if necessary. z.buf[z.raw.start:z.raw.end] remains a contiguous byte
// slice that holds all the bytes read so far for the current token.
// It sets z.err if the underlying reader returns an error.
// Pre-condition: z.err == nil.
func (z *Tokenizer) readByte() byte {
- if z.p1 >= len(z.buf) {
+ if z.raw.end >= len(z.buf) {
// Our buffer is exhausted and we have to read from z.r.
- // We copy z.buf[z.p0:z.p1] to the beginning of z.buf. If the length
- // z.p1 - z.p0 is more than half the capacity of z.buf, then we
+ // We copy z.buf[z.raw.start:z.raw.end] to the beginning of z.buf. If the length
+ // z.raw.end - z.raw.start is more than half the capacity of z.buf, then we
// allocate a new buffer before the copy.
c := cap(z.buf)
- d := z.p1 - z.p0
+ d := z.raw.end - z.raw.start
var buf1 []byte
if 2*d > c {
buf1 = make([]byte, d, 2*c)
} else {
buf1 = z.buf[:d]
}
- copy(buf1, z.buf[z.p0:z.p1])
- z.p0, z.p1, z.buf = 0, d, buf1[:d]
+ copy(buf1, z.buf[z.raw.start:z.raw.end])
+ if x := z.raw.start; x != 0 {
+ // Adjust the data/attr spans to refer to the same contents after the copy.
+ z.data.start -= x
+ z.data.end -= x
+ z.pendingAttr[0].start -= x
+ z.pendingAttr[0].end -= x
+ z.pendingAttr[1].start -= x
+ z.pendingAttr[1].end -= x
+ for i := range z.attr {
+ z.attr[i][0].start -= x
+ z.attr[i][0].end -= x
+ z.attr[i][1].start -= x
+ z.attr[i][1].end -= x
+ }
+ }
+ z.raw.start, z.raw.end, z.buf = 0, d, buf1[:d]
// Now that we have copied the live bytes to the start of the buffer,
// we read from z.r into the remainder.
n, err := z.r.Read(buf1[d:cap(buf1)])
@@ -176,297 +208,467 @@ func (z *Tokenizer) readByte() byte {
}
z.buf = buf1[:d+n]
}
- x := z.buf[z.p1]
- z.p1++
+ x := z.buf[z.raw.end]
+ z.raw.end++
return x
}
-// readTo keeps reading bytes until x is found or a read error occurs. If an
-// error does occur, z.err is set to that error.
-// Pre-condition: z.err == nil.
-func (z *Tokenizer) readTo(x uint8) {
+// skipWhiteSpace skips past any white space.
+func (z *Tokenizer) skipWhiteSpace() {
+ if z.err != nil {
+ return
+ }
for {
c := z.readByte()
if z.err != nil {
return
}
switch c {
- case x:
+ case ' ', '\n', '\r', '\t', '\f':
+ // No-op.
+ default:
+ z.raw.end--
return
- case '\\':
- z.readByte()
+ }
+ }
+}
+
+// readRawOrRCDATA reads until the next "</foo>", where "foo" is z.rawTag and
+// is typically something like "script" or "textarea".
+func (z *Tokenizer) readRawOrRCDATA() {
+loop:
+ for {
+ c := z.readByte()
+ if z.err != nil {
+ break loop
+ }
+ if c != '<' {
+ continue loop
+ }
+ c = z.readByte()
+ if z.err != nil {
+ break loop
+ }
+ if c != '/' {
+ continue loop
+ }
+ for i := 0; i < len(z.rawTag); i++ {
+ c = z.readByte()
if z.err != nil {
- return
+ break loop
+ }
+ if c != z.rawTag[i] && c != z.rawTag[i]-('a'-'A') {
+ continue loop
}
}
+ c = z.readByte()
+ if z.err != nil {
+ break loop
+ }
+ switch c {
+ case ' ', '\n', '\r', '\t', '\f', '/', '>':
+ // The 3 is 2 for the leading "</" plus 1 for the trailing character c.
+ z.raw.end -= 3 + len(z.rawTag)
+ break loop
+ case '<':
+ // Step back one, to catch "</foo</foo>".
+ z.raw.end--
+ }
}
+ z.data.end = z.raw.end
+ // A textarea's or title's RCDATA can contain escaped entities.
+ z.textIsRaw = z.rawTag != "textarea" && z.rawTag != "title"
+ z.rawTag = ""
}
-// nextComment reads the next token starting with "<!--".
-// The opening "<!--" has already been consumed.
-// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 4 <= z.p1.
-func (z *Tokenizer) nextComment() {
- // <!--> is a valid comment.
+// readComment reads the next comment token starting with "<!--". The opening
+// "<!--" has already been consumed.
+func (z *Tokenizer) readComment() {
+ z.data.start = z.raw.end
+ defer func() {
+ if z.data.end < z.data.start {
+ // It's a comment with no data, like <!-->.
+ z.data.end = z.data.start
+ }
+ }()
for dashCount := 2; ; {
c := z.readByte()
if z.err != nil {
+ z.data.end = z.raw.end
return
}
switch c {
case '-':
dashCount++
+ continue
case '>':
if dashCount >= 2 {
- z.tt = CommentToken
+ z.data.end = z.raw.end - len("-->")
return
}
- dashCount = 0
- default:
- dashCount = 0
+ case '!':
+ if dashCount >= 2 {
+ c = z.readByte()
+ if z.err != nil {
+ z.data.end = z.raw.end
+ return
+ }
+ if c == '>' {
+ z.data.end = z.raw.end - len("--!>")
+ return
+ }
+ }
}
+ dashCount = 0
}
}
-// nextMarkupDeclaration reads the next token starting with "<!".
-// It might be a "<!--comment-->", a "<!DOCTYPE foo>", or "<!malformed text".
-// The opening "<!" has already been consumed.
-// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 2 <= z.p1.
-func (z *Tokenizer) nextMarkupDeclaration() {
+// readUntilCloseAngle reads until the next ">".
+func (z *Tokenizer) readUntilCloseAngle() {
+ z.data.start = z.raw.end
+ for {
+ c := z.readByte()
+ if z.err != nil {
+ z.data.end = z.raw.end
+ return
+ }
+ if c == '>' {
+ z.data.end = z.raw.end - len(">")
+ return
+ }
+ }
+}
+
+// readMarkupDeclaration reads the next token starting with "<!". It might be
+// a "<!--comment-->", a "<!DOCTYPE foo>", or "<!a bogus comment". The opening
+// "<!" has already been consumed.
+func (z *Tokenizer) readMarkupDeclaration() TokenType {
+ z.data.start = z.raw.end
var c [2]byte
for i := 0; i < 2; i++ {
c[i] = z.readByte()
if z.err != nil {
- return
+ z.data.end = z.raw.end
+ return CommentToken
}
}
if c[0] == '-' && c[1] == '-' {
- z.nextComment()
- return
+ z.readComment()
+ return CommentToken
}
- z.p1 -= 2
- const s = "DOCTYPE "
- for i := 0; ; i++ {
+ z.raw.end -= 2
+ const s = "DOCTYPE"
+ for i := 0; i < len(s); i++ {
c := z.readByte()
if z.err != nil {
- return
+ z.data.end = z.raw.end
+ return CommentToken
}
- // Capitalize c.
- if 'a' <= c && c <= 'z' {
- c = 'A' + (c - 'a')
- }
- if i < len(s) && c != s[i] {
- z.nextText()
- return
- }
- if c == '>' {
- if i >= len(s) {
- z.tt = DoctypeToken
- }
- return
+ if c != s[i] && c != s[i]+('a'-'A') {
+ // Back up to read the fragment of "DOCTYPE" again.
+ z.raw.end = z.data.start
+ z.readUntilCloseAngle()
+ return CommentToken
}
}
+ if z.skipWhiteSpace(); z.err != nil {
+ z.data.start = z.raw.end
+ z.data.end = z.raw.end
+ return DoctypeToken
+ }
+ z.readUntilCloseAngle()
+ return DoctypeToken
}
-// nextTag reads the next token starting with "<". It might be a "<startTag>",
-// an "</endTag>", a "<!markup declaration>", or "<malformed text".
-// The opening "<" has already been consumed.
-// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1.
-func (z *Tokenizer) nextTag() {
- c := z.readByte()
- if z.err != nil {
- return
- }
- switch {
- case c == '/':
- z.tt = EndTagToken
- // Lower-cased characters are more common in tag names, so we check for them first.
- case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
- z.tt = StartTagToken
- case c == '!':
- z.nextMarkupDeclaration()
- return
- case c == '?':
- z.tt, z.err = ErrorToken, os.NewError("html: TODO: implement XML processing instructions")
- return
- default:
- z.tt, z.err = ErrorToken, os.NewError("html: TODO: handle malformed tags")
- return
+// readStartTag reads the next start tag token. The opening "<a" has already
+// been consumed, where 'a' means anything in [A-Za-z].
+func (z *Tokenizer) readStartTag() TokenType {
+ z.attr = z.attr[:0]
+ z.nAttrReturned = 0
+ // Read the tag name and attribute key/value pairs.
+ z.readTagName()
+ if z.skipWhiteSpace(); z.err != nil {
+ return ErrorToken
}
for {
c := z.readByte()
- if z.err != nil {
- return
+ if z.err != nil || c == '>' {
+ break
}
- switch c {
- case '"', '\'':
- z.readTo(c)
- if z.err != nil {
- return
- }
- case '>':
- if z.buf[z.p1-2] == '/' && z.tt == StartTagToken {
- z.tt = SelfClosingTagToken
+ z.raw.end--
+ z.readTagAttrKey()
+ z.readTagAttrVal()
+ // Save pendingAttr if it has a non-empty key.
+ if z.pendingAttr[0].start != z.pendingAttr[0].end {
+ z.attr = append(z.attr, z.pendingAttr)
+ }
+ if z.skipWhiteSpace(); z.err != nil {
+ break
+ }
+ }
+ // Any "<noembed>", "<noframes>", "<noscript>", "<script>", "<style>",
+ // "<textarea>" or "<title>" tag flags the tokenizer's next token as raw.
+ // The tag name lengths of these special cases ranges in [5, 8].
+ if x := z.data.end - z.data.start; 5 <= x && x <= 8 {
+ switch z.buf[z.data.start] {
+ case 'n', 's', 't', 'N', 'S', 'T':
+ switch s := strings.ToLower(string(z.buf[z.data.start:z.data.end])); s {
+ case "noembed", "noframes", "noscript", "script", "style", "textarea", "title":
+ z.rawTag = s
}
+ }
+ }
+ // Look for a self-closing token like "<br/>".
+ if z.err == nil && z.buf[z.raw.end-2] == '/' {
+ return SelfClosingTagToken
+ }
+ return StartTagToken
+}
+
+// readEndTag reads the next end tag token. The opening "</a" has already
+// been consumed, where 'a' means anything in [A-Za-z].
+func (z *Tokenizer) readEndTag() {
+ z.attr = z.attr[:0]
+ z.nAttrReturned = 0
+ z.readTagName()
+ for {
+ c := z.readByte()
+ if z.err != nil || c == '>' {
return
}
}
}
-// nextText reads all text up until an '<'.
-// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1.
-func (z *Tokenizer) nextText() {
+// readTagName sets z.data to the "div" in "<div k=v>". The reader (z.raw.end)
+// is positioned such that the first byte of the tag name (the "d" in "<div")
+// has already been consumed.
+func (z *Tokenizer) readTagName() {
+ z.data.start = z.raw.end - 1
for {
c := z.readByte()
if z.err != nil {
+ z.data.end = z.raw.end
return
}
- if c == '<' {
- z.p1--
+ switch c {
+ case ' ', '\n', '\r', '\t', '\f':
+ z.data.end = z.raw.end - 1
+ return
+ case '/', '>':
+ z.raw.end--
+ z.data.end = z.raw.end
return
}
}
}
-// Next scans the next token and returns its type.
-func (z *Tokenizer) Next() TokenType {
+// readTagAttrKey sets z.pendingAttr[0] to the "k" in "<div k=v>".
+// Precondition: z.err == nil.
+func (z *Tokenizer) readTagAttrKey() {
+ z.pendingAttr[0].start = z.raw.end
for {
- if z.err != nil {
- z.tt = ErrorToken
- return z.tt
- }
- z.p0 = z.p1
c := z.readByte()
if z.err != nil {
- z.tt = ErrorToken
- return z.tt
+ z.pendingAttr[0].end = z.raw.end
+ return
}
- // We assume that the next token is text unless proven otherwise.
- z.tt = TextToken
- if c != '<' {
- z.nextText()
- } else {
- z.nextTag()
- if z.tt == CommentToken && !z.ReturnComments {
- continue
- }
+ switch c {
+ case ' ', '\n', '\r', '\t', '\f', '/':
+ z.pendingAttr[0].end = z.raw.end - 1
+ return
+ case '=', '>':
+ z.raw.end--
+ z.pendingAttr[0].end = z.raw.end
+ return
}
- return z.tt
}
- panic("unreachable")
}
-// trim returns the largest j such that z.buf[i:j] contains only white space,
-// or only white space plus the final ">" or "/>" of the raw data.
-func (z *Tokenizer) trim(i int) int {
- k := z.p1
- for ; i < k; i++ {
- switch z.buf[i] {
- case ' ', '\n', '\t', '\f':
- continue
- case '>':
- if i == k-1 {
- return k
+// readTagAttrVal sets z.pendingAttr[1] to the "v" in "<div k=v>".
+func (z *Tokenizer) readTagAttrVal() {
+ z.pendingAttr[1].start = z.raw.end
+ z.pendingAttr[1].end = z.raw.end
+ if z.skipWhiteSpace(); z.err != nil {
+ return
+ }
+ c := z.readByte()
+ if z.err != nil {
+ return
+ }
+ if c != '=' {
+ z.raw.end--
+ return
+ }
+ if z.skipWhiteSpace(); z.err != nil {
+ return
+ }
+ quote := z.readByte()
+ if z.err != nil {
+ return
+ }
+ switch quote {
+ case '>':
+ z.raw.end--
+ return
+
+ case '\'', '"':
+ z.pendingAttr[1].start = z.raw.end
+ for {
+ c := z.readByte()
+ if z.err != nil {
+ z.pendingAttr[1].end = z.raw.end
+ return
}
- case '/':
- if i == k-2 {
- return k
+ if c == quote {
+ z.pendingAttr[1].end = z.raw.end - 1
+ return
}
}
- return i
- }
- return k
-}
-// tagName finds the tag name at the start of z.buf[i:] and returns that name
-// lower-cased, as well as the trimmed cursor location afterwards.
-func (z *Tokenizer) tagName(i int) ([]byte, int) {
- i0 := i
-loop:
- for ; i < z.p1; i++ {
- c := z.buf[i]
- switch c {
- case ' ', '\n', '\t', '\f', '/', '>':
- break loop
- }
- if 'A' <= c && c <= 'Z' {
- z.buf[i] = c + 'a' - 'A'
+ default:
+ z.pendingAttr[1].start = z.raw.end - 1
+ for {
+ c := z.readByte()
+ if z.err != nil {
+ z.pendingAttr[1].end = z.raw.end
+ return
+ }
+ switch c {
+ case ' ', '\n', '\r', '\t', '\f':
+ z.pendingAttr[1].end = z.raw.end - 1
+ return
+ case '>':
+ z.raw.end--
+ z.pendingAttr[1].end = z.raw.end
+ return
+ }
}
}
- return z.buf[i0:i], z.trim(i)
}
-// unquotedAttrVal finds the unquoted attribute value at the start of z.buf[i:]
-// and returns that value, as well as the trimmed cursor location afterwards.
-func (z *Tokenizer) unquotedAttrVal(i int) ([]byte, int) {
- i0 := i
+// next scans the next token and returns its type.
+func (z *Tokenizer) next() TokenType {
+ if z.err != nil {
+ return ErrorToken
+ }
+ z.raw.start = z.raw.end
+ z.data.start = z.raw.end
+ z.data.end = z.raw.end
+ if z.rawTag != "" {
+ z.readRawOrRCDATA()
+ return TextToken
+ }
+ z.textIsRaw = false
+
loop:
- for ; i < z.p1; i++ {
- switch z.buf[i] {
- case ' ', '\n', '\t', '\f', '>':
+ for {
+ c := z.readByte()
+ if z.err != nil {
break loop
- case '&':
- // TODO: unescape the entity.
}
- }
- return z.buf[i0:i], z.trim(i)
-}
-
-// attrName finds the largest attribute name at the start
-// of z.buf[i:] and returns it lower-cased, as well
-// as the trimmed cursor location after that name.
-//
-// http://dev.w3.org/html5/spec/Overview.html#syntax-attribute-name
-// TODO: unicode characters
-func (z *Tokenizer) attrName(i int) ([]byte, int) {
- for z.buf[i] == '/' {
- i++
- if z.buf[i] == '>' {
- return nil, z.trim(i)
+ if c != '<' {
+ continue loop
}
- }
- i0 := i
-loop:
- for ; i < z.p1; i++ {
- c := z.buf[i]
- switch c {
- case '>', '/', '=':
+
+ // Check if the '<' we have just read is part of a tag, comment
+ // or doctype. If not, it's part of the accumulated text token.
+ c = z.readByte()
+ if z.err != nil {
break loop
}
+ var tokenType TokenType
switch {
- case 'A' <= c && c <= 'Z':
- z.buf[i] = c + 'a' - 'A'
- case c > ' ' && c < 0x7f:
- // No-op.
+ case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
+ tokenType = StartTagToken
+ case c == '/':
+ tokenType = EndTagToken
+ case c == '!' || c == '?':
+ // We use CommentToken to mean any of "<!--actual comments-->",
+ // "<!DOCTYPE declarations>" and "<?xml processing instructions?>".
+ tokenType = CommentToken
default:
- break loop
+ continue
+ }
+
+ // We have a non-text token, but we might have accumulated some text
+ // before that. If so, we return the text first, and return the non-
+ // text token on the subsequent call to Next.
+ if x := z.raw.end - len("<a"); z.raw.start < x {
+ z.raw.end = x
+ z.data.end = x
+ return TextToken
+ }
+ switch tokenType {
+ case StartTagToken:
+ return z.readStartTag()
+ case EndTagToken:
+ c = z.readByte()
+ if z.err != nil {
+ break loop
+ }
+ if c == '>' {
+ // "</>" does not generate a token at all.
+ // Reset the tokenizer state and start again.
+ z.raw.start = z.raw.end
+ z.data.start = z.raw.end
+ z.data.end = z.raw.end
+ continue loop
+ }
+ if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
+ z.readEndTag()
+ return EndTagToken
+ }
+ z.raw.end--
+ z.readUntilCloseAngle()
+ return CommentToken
+ case CommentToken:
+ if c == '!' {
+ return z.readMarkupDeclaration()
+ }
+ z.raw.end--
+ z.readUntilCloseAngle()
+ return CommentToken
+ }
+ }
+ if z.raw.start < z.raw.end {
+ z.data.end = z.raw.end
+ return TextToken
+ }
+ return ErrorToken
+}
+
+// Next scans the next token and returns its type.
+func (z *Tokenizer) Next() TokenType {
+ for {
+ z.tt = z.next()
+ // TODO: remove the ReturnComments option. A tokenizer should
+ // always return comment tags.
+ if z.tt == CommentToken && !z.ReturnComments {
+ continue
}
+ return z.tt
}
- return z.buf[i0:i], z.trim(i)
+ panic("unreachable")
+}
+
+// Raw returns the unmodified text of the current token. Calling Next, Token,
+// Text, TagName or TagAttr may change the contents of the returned slice.
+func (z *Tokenizer) Raw() []byte {
+ return z.buf[z.raw.start:z.raw.end]
}
// Text returns the unescaped text of a text, comment or doctype token. The
// contents of the returned slice may change on the next call to Next.
func (z *Tokenizer) Text() []byte {
- var i0, i1 int
switch z.tt {
- case TextToken:
- i0 = z.p0
- i1 = z.p1
- case CommentToken:
- // Trim the "<!--" from the left and the "-->" from the right.
- // "<!-->" is a valid comment, so the adjusted endpoints might overlap.
- i0 = z.p0 + 4
- i1 = z.p1 - 3
- case DoctypeToken:
- // Trim the "<!DOCTYPE " from the left and the ">" from the right.
- i0 = z.p0 + 10
- i1 = z.p1 - 1
- default:
- return nil
- }
- z.p0 = z.p1
- if i0 < i1 {
- return unescape(z.buf[i0:i1])
+ case TextToken, CommentToken, DoctypeToken:
+ s := z.buf[z.data.start:z.data.end]
+ z.data.start = z.raw.end
+ z.data.end = z.raw.end
+ if !z.textIsRaw {
+ s = unescape(s)
+ }
+ return s
}
return nil
}
@@ -475,73 +677,33 @@ func (z *Tokenizer) Text() []byte {
// `<IMG SRC="foo">`) and whether the tag has attributes.
// The contents of the returned slice may change on the next call to Next.
func (z *Tokenizer) TagName() (name []byte, hasAttr bool) {
- i := z.p0 + 1
- if i >= z.p1 {
- z.p0 = z.p1
- return nil, false
- }
- if z.buf[i] == '/' {
- i++
+ if z.data.start < z.data.end {
+ switch z.tt {
+ case StartTagToken, EndTagToken, SelfClosingTagToken:
+ s := z.buf[z.data.start:z.data.end]
+ z.data.start = z.raw.end
+ z.data.end = z.raw.end
+ return lower(s), z.nAttrReturned < len(z.attr)
+ }
}
- name, z.p0 = z.tagName(i)
- hasAttr = z.p0 != z.p1
- return
+ return nil, false
}
// TagAttr returns the lower-cased key and unescaped value of the next unparsed
// attribute for the current tag token and whether there are more attributes.
// The contents of the returned slices may change on the next call to Next.
func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) {
- key, i := z.attrName(z.p0)
- // Check for an empty attribute value.
- if i == z.p1 {
- z.p0 = i
- return
- }
- // Get past the equals and quote characters.
- if z.buf[i] != '=' {
- z.p0, moreAttr = i, true
- return
- }
- i = z.trim(i + 1)
- if i == z.p1 {
- z.p0 = i
- return
- }
- closeQuote := z.buf[i]
- if closeQuote != '\'' && closeQuote != '"' {
- val, z.p0 = z.unquotedAttrVal(i)
- moreAttr = z.p0 != z.p1
- return
- }
- i = z.trim(i + 1)
- // Copy and unescape everything up to the closing quote.
- dst, src := i, i
-loop:
- for src < z.p1 {
- c := z.buf[src]
- switch c {
- case closeQuote:
- src++
- break loop
- case '&':
- dst, src = unescapeEntity(z.buf, dst, src, true)
- case '\\':
- if src == z.p1 {
- z.buf[dst] = '\\'
- dst++
- } else {
- z.buf[dst] = z.buf[src+1]
- dst, src = dst+1, src+2
- }
- default:
- z.buf[dst] = c
- dst, src = dst+1, src+1
+ if z.nAttrReturned < len(z.attr) {
+ switch z.tt {
+ case StartTagToken, SelfClosingTagToken:
+ x := z.attr[z.nAttrReturned]
+ z.nAttrReturned++
+ key = z.buf[x[0].start:x[0].end]
+ val = z.buf[x[1].start:x[1].end]
+ return lower(key), unescape(val), z.nAttrReturned < len(z.attr)
}
}
- val, z.p0 = z.buf[i:dst], z.trim(src)
- moreAttr = z.p0 != z.p1
- return
+ return nil, nil, false
}
// Token returns the next Token. The result's Data and Attr values remain valid
@@ -551,7 +713,7 @@ func (z *Tokenizer) Token() Token {
switch z.tt {
case TextToken, CommentToken, DoctypeToken:
t.Data = string(z.Text())
- case StartTagToken, EndTagToken, SelfClosingTagToken:
+ case StartTagToken, SelfClosingTagToken:
var attr []Attribute
name, moreAttr := z.TagName()
for moreAttr {
@@ -561,6 +723,9 @@ func (z *Tokenizer) Token() Token {
}
t.Data = string(name)
t.Attr = attr
+ case EndTagToken:
+ name, _ := z.TagName()
+ t.Data = string(name)
}
return t
}
View Lorry Controller Status