diff options
Diffstat (limited to 'libgo/go/go/parser')
| -rw-r--r-- | libgo/go/go/parser/error_test.go | 166 | ||||
| -rw-r--r-- | libgo/go/go/parser/parser.go | 152 | ||||
| -rw-r--r-- | libgo/go/go/parser/parser_test.go | 83 | ||||
| -rw-r--r-- | libgo/go/go/parser/short_test.go | 75 | ||||
| -rw-r--r-- | libgo/go/go/parser/testdata/commas.src | 19 | ||||
| -rw-r--r-- | libgo/go/go/parser/testdata/issue3106.src | 46 |
6 files changed, 435 insertions, 106 deletions
diff --git a/libgo/go/go/parser/error_test.go b/libgo/go/go/parser/error_test.go new file mode 100644 index 00000000000..377c8b80cba --- /dev/null +++ b/libgo/go/go/parser/error_test.go @@ -0,0 +1,166 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a parser test harness. The files in the testdata +// directory are parsed and the errors reported are compared against the +// error messages expected in the test files. The test files must end in +// .src rather than .go so that they are not disturbed by gofmt runs. +// +// Expected errors are indicated in the test files by putting a comment +// of the form /* ERROR "rx" */ immediately following an offending token. +// The harness will verify that an error matching the regular expression +// rx is reported at that source position. +// +// For instance, the following test file indicates that a "not declared" +// error should be reported for the undeclared variable x: +// +// package p +// func f() { +// _ = x /* ERROR "not declared" */ + 1 +// } + +package parser + +import ( + "go/scanner" + "go/token" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + "testing" +) + +const testdata = "testdata" + +// getFile assumes that each filename occurs at most once +func getFile(filename string) (file *token.File) { + fset.Iterate(func(f *token.File) bool { + if f.Name() == filename { + if file != nil { + panic(filename + " used multiple times") + } + file = f + } + return true + }) + return file +} + +func getPos(filename string, offset int) token.Pos { + if f := getFile(filename); f != nil { + return f.Pos(offset) + } + return token.NoPos +} + +// ERROR comments must be of the form /* ERROR "rx" */ and rx is +// a regular expression that matches the expected error message. +// +var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`) + +// expectedErrors collects the regular expressions of ERROR comments found +// in files and returns them as a map of error positions to error messages. +// +func expectedErrors(t *testing.T, filename string, src []byte) map[token.Pos]string { + errors := make(map[token.Pos]string) + + var s scanner.Scanner + // file was parsed already - do not add it again to the file + // set otherwise the position information returned here will + // not match the position information collected by the parser + s.Init(getFile(filename), src, nil, scanner.ScanComments) + var prev token.Pos // position of last non-comment, non-semicolon token + + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + return errors + case token.COMMENT: + s := errRx.FindStringSubmatch(lit) + if len(s) == 2 { + errors[prev] = string(s[1]) + } + default: + prev = pos + } + } + + panic("unreachable") +} + +// compareErrors compares the map of expected error messages with the list +// of found errors and reports discrepancies. +// +func compareErrors(t *testing.T, expected map[token.Pos]string, found scanner.ErrorList) { + for _, error := range found { + // error.Pos is a token.Position, but we want + // a token.Pos so we can do a map lookup + pos := getPos(error.Pos.Filename, error.Pos.Offset) + if msg, found := expected[pos]; found { + // we expect a message at pos; check if it matches + rx, err := regexp.Compile(msg) + if err != nil { + t.Errorf("%s: %v", error.Pos, err) + continue + } + if match := rx.MatchString(error.Msg); !match { + t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg) + continue + } + // we have a match - eliminate this error + delete(expected, pos) + } else { + // To keep in mind when analyzing failed test output: + // If the same error position occurs multiple times in errors, + // this message will be triggered (because the first error at + // the position removes this position from the expected errors). + t.Errorf("%s: unexpected error: %s", error.Pos, error.Msg) + } + } + + // there should be no expected errors left + if len(expected) > 0 { + t.Errorf("%d errors not reported:", len(expected)) + for pos, msg := range expected { + t.Errorf("%s: %s\n", fset.Position(pos), msg) + } + } +} + +func checkErrors(t *testing.T, filename string, input interface{}) { + src, err := readSource(filename, input) + if err != nil { + t.Error(err) + return + } + + _, err = ParseFile(fset, filename, src, DeclarationErrors) + found, ok := err.(scanner.ErrorList) + if err != nil && !ok { + t.Error(err) + return + } + + // we are expecting the following errors + // (collect these after parsing a file so that it is found in the file set) + expected := expectedErrors(t, filename, src) + + // verify errors returned by the parser + compareErrors(t, expected, found) +} + +func TestErrors(t *testing.T) { + list, err := ioutil.ReadDir(testdata) + if err != nil { + t.Fatal(err) + } + for _, fi := range list { + name := fi.Name() + if !fi.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".src") { + checkErrors(t, filepath.Join(testdata, name), nil) + } + } +} diff --git a/libgo/go/go/parser/parser.go b/libgo/go/go/parser/parser.go index a122baf0879..e362e13a7b6 100644 --- a/libgo/go/go/parser/parser.go +++ b/libgo/go/go/parser/parser.go @@ -40,6 +40,13 @@ type parser struct { tok token.Token // one token look-ahead lit string // token literal + // Error recovery + // (used to limit the number of calls to syncXXX functions + // w/o making scanning progress - avoids potential endless + // loops across multiple parser functions during error recovery) + syncPos token.Pos // last synchronization position + syncCnt int // number of calls to syncXXX without progress + // Non-syntactic parser control exprLev int // < 0: in control clause, >= 0: in expression @@ -362,18 +369,36 @@ func (p *parser) expect(tok token.Token) token.Pos { // expectClosing is like expect but provides a better error message // for the common case of a missing comma before a newline. // -func (p *parser) expectClosing(tok token.Token, construct string) token.Pos { +func (p *parser) expectClosing(tok token.Token, context string) token.Pos { if p.tok != tok && p.tok == token.SEMICOLON && p.lit == "\n" { - p.error(p.pos, "missing ',' before newline in "+construct) + p.error(p.pos, "missing ',' before newline in "+context) p.next() } return p.expect(tok) } func (p *parser) expectSemi() { + // semicolon is optional before a closing ')' or '}' if p.tok != token.RPAREN && p.tok != token.RBRACE { - p.expect(token.SEMICOLON) + if p.tok == token.SEMICOLON { + p.next() + } else { + p.errorExpected(p.pos, "';'") + syncStmt(p) + } + } +} + +func (p *parser) atComma(context string) bool { + if p.tok == token.COMMA { + return true + } + if p.tok == token.SEMICOLON && p.lit == "\n" { + p.error(p.pos, "missing ',' before newline in "+context) + return true // "insert" the comma and continue + } + return false } func assert(cond bool, msg string) { @@ -382,6 +407,68 @@ func assert(cond bool, msg string) { } } +// syncStmt advances to the next statement. +// Used for synchronization after an error. +// +func syncStmt(p *parser) { + for { + switch p.tok { + case token.BREAK, token.CONST, token.CONTINUE, token.DEFER, + token.FALLTHROUGH, token.FOR, token.GO, token.GOTO, + token.IF, token.RETURN, token.SELECT, token.SWITCH, + token.TYPE, token.VAR: + // Return only if parser made some progress since last + // sync or if it has not reached 10 sync calls without + // progress. Otherwise consume at least one token to + // avoid an endless parser loop (it is possible that + // both parseOperand and parseStmt call syncStmt and + // correctly do not advance, thus the need for the + // invocation limit p.syncCnt). + if p.pos == p.syncPos && p.syncCnt < 10 { + p.syncCnt++ + return + } + if p.pos > p.syncPos { + p.syncPos = p.pos + p.syncCnt = 0 + return + } + // Reaching here indicates a parser bug, likely an + // incorrect token list in this function, but it only + // leads to skipping of possibly correct code if a + // previous error is present, and thus is preferred + // over a non-terminating parse. + case token.EOF: + return + } + p.next() + } +} + +// syncDecl advances to the next declaration. +// Used for synchronization after an error. +// +func syncDecl(p *parser) { + for { + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + // see comments in syncStmt + if p.pos == p.syncPos && p.syncCnt < 10 { + p.syncCnt++ + return + } + if p.pos > p.syncPos { + p.syncPos = p.pos + p.syncCnt = 0 + return + } + case token.EOF: + return + } + p.next() + } +} + // ---------------------------------------------------------------------------- // Identifiers @@ -522,9 +609,11 @@ func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident { for i, x := range list { ident, isIdent := x.(*ast.Ident) if !isIdent { - pos := x.Pos() - p.errorExpected(pos, "identifier") - ident = &ast.Ident{NamePos: pos, Name: "_"} + if _, isBad := x.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.errorExpected(x.Pos(), "identifier") + } + ident = &ast.Ident{NamePos: x.Pos(), Name: "_"} } idents[i] = ident } @@ -688,7 +777,7 @@ func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params [ // Go spec: The scope of an identifier denoting a function // parameter or result variable is the function body. p.declare(field, nil, scope, ast.Var, idents...) - if p.tok != token.COMMA { + if !p.atComma("parameter list") { break } p.next() @@ -991,19 +1080,19 @@ func (p *parser) parseOperand(lhs bool) ast.Expr { case token.FUNC: return p.parseFuncTypeOrLit() + } - default: - if typ := p.tryIdentOrType(true); typ != nil { - // could be type for composite literal or conversion - _, isIdent := typ.(*ast.Ident) - assert(!isIdent, "type cannot be identifier") - return typ - } + if typ := p.tryIdentOrType(true); typ != nil { + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ } + // we have an error pos := p.pos p.errorExpected(pos, "operand") - p.next() // make progress + syncStmt(p) return &ast.BadExpr{From: pos, To: p.pos} } @@ -1078,7 +1167,7 @@ func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { ellipsis = p.pos p.next() } - if p.tok != token.COMMA { + if !p.atComma("argument list") { break } p.next() @@ -1118,7 +1207,7 @@ func (p *parser) parseElementList() (list []ast.Expr) { for p.tok != token.RBRACE && p.tok != token.EOF { list = append(list, p.parseElement(true)) - if p.tok != token.COMMA { + if !p.atComma("composite literal") { break } p.next() @@ -1262,8 +1351,8 @@ L: x = p.parseTypeAssertion(p.checkExpr(x)) default: pos := p.pos - p.next() // make progress p.errorExpected(pos, "selector or type assertion") + p.next() // make progress x = &ast.BadExpr{From: pos, To: p.pos} } case token.LBRACK: @@ -1471,7 +1560,10 @@ func (p *parser) parseCallExpr() *ast.CallExpr { if call, isCall := x.(*ast.CallExpr); isCall { return call } - p.errorExpected(x.Pos(), "function/method call") + if _, isBad := x.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.errorExpected(x.Pos(), "function/method call") + } return nil } @@ -1862,7 +1954,7 @@ func (p *parser) parseStmt() (s ast.Stmt) { switch p.tok { case token.CONST, token.TYPE, token.VAR: - s = &ast.DeclStmt{Decl: p.parseDecl()} + s = &ast.DeclStmt{Decl: p.parseDecl(syncStmt)} case // tokens that may start an expression token.IDENT, token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operands @@ -1904,7 +1996,7 @@ func (p *parser) parseStmt() (s ast.Stmt) { // no statement found pos := p.pos p.errorExpected(pos, "statement") - p.next() // make progress + syncStmt(p) s = &ast.BadStmt{From: pos, To: p.pos} } @@ -2095,8 +2187,13 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { recv := par.List[0] base := deref(recv.Type) if _, isIdent := base.(*ast.Ident); !isIdent { - p.errorExpected(base.Pos(), "(unqualified) identifier") - par.List = []*ast.Field{{Type: &ast.BadExpr{From: recv.Pos(), To: recv.End()}}} + if _, isBad := base.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.errorExpected(base.Pos(), "(unqualified) identifier") + } + par.List = []*ast.Field{ + {Type: &ast.BadExpr{From: recv.Pos(), To: recv.End()}}, + } } return par @@ -2152,7 +2249,7 @@ func (p *parser) parseFuncDecl() *ast.FuncDecl { return decl } -func (p *parser) parseDecl() ast.Decl { +func (p *parser) parseDecl(sync func(*parser)) ast.Decl { if p.trace { defer un(trace(p, "Declaration")) } @@ -2174,9 +2271,8 @@ func (p *parser) parseDecl() ast.Decl { default: pos := p.pos p.errorExpected(pos, "declaration") - p.next() // make progress - decl := &ast.BadDecl{From: pos, To: p.pos} - return decl + sync(p) + return &ast.BadDecl{From: pos, To: p.pos} } return p.parseGenDecl(p.tok, f) @@ -2215,7 +2311,7 @@ func (p *parser) parseFile() *ast.File { if p.mode&ImportsOnly == 0 { // rest of package body for p.tok != token.EOF { - decls = append(decls, p.parseDecl()) + decls = append(decls, p.parseDecl(syncDecl)) } } } diff --git a/libgo/go/go/parser/parser_test.go b/libgo/go/go/parser/parser_test.go index 93ca3d6aa39..5e45acd007a 100644 --- a/libgo/go/go/parser/parser_test.go +++ b/libgo/go/go/parser/parser_test.go @@ -14,87 +14,14 @@ import ( var fset = token.NewFileSet() -var illegalInputs = []interface{}{ - nil, - 3.14, - []byte(nil), - "foo!", - `package p; func f() { if /* should have condition */ {} };`, - `package p; func f() { if ; /* should have condition */ {} };`, - `package p; func f() { if f(); /* should have condition */ {} };`, - `package p; const c; /* should have constant value */`, - `package p; func f() { if _ = range x; true {} };`, - `package p; func f() { switch _ = range x; true {} };`, - `package p; func f() { for _ = range x ; ; {} };`, - `package p; func f() { for ; ; _ = range x {} };`, - `package p; func f() { for ; _ = range x ; {} };`, - `package p; func f() { switch t = t.(type) {} };`, - `package p; func f() { switch t, t = t.(type) {} };`, - `package p; func f() { switch t = t.(type), t {} };`, - `package p; var a = [1]int; /* illegal expression */`, - `package p; var a = [...]int; /* illegal expression */`, - `package p; var a = struct{} /* illegal expression */`, - `package p; var a = func(); /* illegal expression */`, - `package p; var a = interface{} /* illegal expression */`, - `package p; var a = []int /* illegal expression */`, - `package p; var a = map[int]int /* illegal expression */`, - `package p; var a = chan int; /* illegal expression */`, - `package p; var a = []int{[]int}; /* illegal expression */`, - `package p; var a = ([]int); /* illegal expression */`, - `package p; var a = a[[]int:[]int]; /* illegal expression */`, - `package p; var a = <- chan int; /* illegal expression */`, - `package p; func f() { select { case _ <- chan int: } };`, -} - -func TestParseIllegalInputs(t *testing.T) { - for _, src := range illegalInputs { - _, err := ParseFile(fset, "", src, 0) - if err == nil { - t.Errorf("ParseFile(%v) should have failed", src) - } - } -} - -var validPrograms = []string{ - "package p\n", - `package p;`, - `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, - `package p; func f() { if f(T{}) {} };`, - `package p; func f() { _ = (<-chan int)(x) };`, - `package p; func f() { _ = (<-chan <-chan int)(x) };`, - `package p; func f(func() func() func());`, - `package p; func f(...T);`, - `package p; func f(float, ...int);`, - `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, - `package p; func f(int,) {};`, - `package p; func f(...int,) {};`, - `package p; func f(x ...int,) {};`, - `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, - `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, - `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, - `package p; var a = T{{1, 2}, {3, 4}}`, - `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, - `package p; func f() { select { case x := (<-c): } };`, - `package p; func f() { if ; true {} };`, - `package p; func f() { switch ; {} };`, - `package p; func f() { for _ = range "foo" + "bar" {} };`, -} - -func TestParseValidPrograms(t *testing.T) { - for _, src := range validPrograms { - _, err := ParseFile(fset, "", src, SpuriousErrors) - if err != nil { - t.Errorf("ParseFile(%q): %v", src, err) - } - } -} - var validFiles = []string{ "parser.go", "parser_test.go", + "error_test.go", + "short_test.go", } -func TestParse3(t *testing.T) { +func TestParse(t *testing.T) { for _, filename := range validFiles { _, err := ParseFile(fset, filename, nil, DeclarationErrors) if err != nil { @@ -116,7 +43,7 @@ func nameFilter(filename string) bool { func dirFilter(f os.FileInfo) bool { return nameFilter(f.Name()) } -func TestParse4(t *testing.T) { +func TestParseDir(t *testing.T) { path := "." pkgs, err := ParseDir(fset, path, dirFilter, 0) if err != nil { @@ -158,7 +85,7 @@ func TestParseExpr(t *testing.T) { } // it must not crash - for _, src := range validPrograms { + for _, src := range valids { ParseExpr(src) } } diff --git a/libgo/go/go/parser/short_test.go b/libgo/go/go/parser/short_test.go new file mode 100644 index 00000000000..238492bf3f7 --- /dev/null +++ b/libgo/go/go/parser/short_test.go @@ -0,0 +1,75 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for short valid and invalid programs. + +package parser + +import "testing" + +var valids = []string{ + "package p\n", + `package p;`, + `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, + `package p; func f() { if f(T{}) {} };`, + `package p; func f() { _ = (<-chan int)(x) };`, + `package p; func f() { _ = (<-chan <-chan int)(x) };`, + `package p; func f(func() func() func());`, + `package p; func f(...T);`, + `package p; func f(float, ...int);`, + `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; func f(int,) {};`, + `package p; func f(...int,) {};`, + `package p; func f(x ...int,) {};`, + `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, + `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, + `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, + `package p; var a = T{{1, 2}, {3, 4}}`, + `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { select { case x := (<-c): } };`, + `package p; func f() { if ; true {} };`, + `package p; func f() { switch ; {} };`, + `package p; func f() { for _ = range "foo" + "bar" {} };`, +} + +func TestValid(t *testing.T) { + for _, src := range valids { + checkErrors(t, src, src) + } +} + +var invalids = []string{ + `foo /* ERROR "expected 'package'" */ !`, + `package p; func f() { if { /* ERROR "expected operand" */ } };`, + `package p; func f() { if ; { /* ERROR "expected operand" */ } };`, + `package p; func f() { if f(); { /* ERROR "expected operand" */ } };`, + `package p; const c; /* ERROR "expected '='" */`, + `package p; func f() { if _ /* ERROR "expected condition" */ = range x; true {} };`, + `package p; func f() { switch _ /* ERROR "expected condition" */ = range x; true {} };`, + `package p; func f() { for _ = range x ; /* ERROR "expected '{'" */ ; {} };`, + `package p; func f() { for ; ; _ = range /* ERROR "expected operand" */ x {} };`, + `package p; func f() { for ; _ /* ERROR "expected condition" */ = range x ; {} };`, + `package p; func f() { switch t /* ERROR "expected condition" */ = t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected condition" */ , t = t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected condition" */ = t.(type), t {} };`, + `package p; var a = [ /* ERROR "expected expression" */ 1]int;`, + `package p; var a = [ /* ERROR "expected expression" */ ...]int;`, + `package p; var a = struct /* ERROR "expected expression" */ {}`, + `package p; var a = func /* ERROR "expected expression" */ ();`, + `package p; var a = interface /* ERROR "expected expression" */ {}`, + `package p; var a = [ /* ERROR "expected expression" */ ]int`, + `package p; var a = map /* ERROR "expected expression" */ [int]int`, + `package p; var a = chan /* ERROR "expected expression" */ int;`, + `package p; var a = []int{[ /* ERROR "expected expression" */ ]int};`, + `package p; var a = ( /* ERROR "expected expression" */ []int);`, + `package p; var a = a[[ /* ERROR "expected expression" */ ]int:[]int];`, + `package p; var a = <- /* ERROR "expected expression" */ chan int;`, + `package p; func f() { select { case _ <- chan /* ERROR "expected expression" */ int: } };`, +} + +func TestInvalid(t *testing.T) { + for _, src := range invalids { + checkErrors(t, src, src) + } +} diff --git a/libgo/go/go/parser/testdata/commas.src b/libgo/go/go/parser/testdata/commas.src new file mode 100644 index 00000000000..af6e7064506 --- /dev/null +++ b/libgo/go/go/parser/testdata/commas.src @@ -0,0 +1,19 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for error messages/parser synchronization +// after missing commas. + +package p + +var _ = []int{ + 0 /* ERROR "missing ','" */ +} + +var _ = []int{ + 0, + 1, + 2, + 3 /* ERROR "missing ','" */ +} diff --git a/libgo/go/go/parser/testdata/issue3106.src b/libgo/go/go/parser/testdata/issue3106.src new file mode 100644 index 00000000000..82796c8ceb6 --- /dev/null +++ b/libgo/go/go/parser/testdata/issue3106.src @@ -0,0 +1,46 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 3106: Better synchronization of +// parser after certain syntax errors. + +package main + +func f() { + var m Mutex + c := MakeCond(&m) + percent := 0 + const step = 10 + for i := 0; i < 5; i++ { + go func() { + for { + // Emulates some useful work. + time.Sleep(1e8) + m.Lock() + defer + if /* ERROR "expected operand, found 'if'" */ percent == 100 { + m.Unlock() + break + } + percent++ + if percent % step == 0 { + //c.Signal() + } + m.Unlock() + } + }() + } + for { + m.Lock() + if percent == 0 || percent % step != 0 { + c.Wait() + } + fmt.Print(",") + if percent == 100 { + m.Unlock() + break + } + m.Unlock() + } +} |