From 3b2bc49486d98ef1eb104dfbc2f82dd20f05f7b3 Mon Sep 17 00:00:00 2001 From: Dominik Honnef Date: Wed, 3 Dec 2014 10:28:54 +1100 Subject: cmd/go: regenerate doc.go Move change from CL 170770043 to correct file and regenerate docs for changes from CL 164120043. LGTM=adg R=golang-codereviews, adg, bradfitz CC=golang-codereviews https://codereview.appspot.com/183000043 Committer: Andrew Gerrand --- src/cmd/go/doc.go | 7 ++++++- src/cmd/go/generate.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'src/cmd/go') diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 43a315944..879fc7f8b 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -317,7 +317,7 @@ Download and install packages and dependencies Usage: - go get [-d] [-fix] [-t] [-u] [build flags] [packages] + go get [-d] [-f] [-fix] [-t] [-u] [build flags] [packages] Get downloads and installs the packages named by the import paths, along with their dependencies. @@ -325,6 +325,11 @@ along with their dependencies. The -d flag instructs get to stop after downloading the packages; that is, it instructs get not to install the packages. +The -f flag, valid only when -u is set, forces get -u not to verify that +each package has been checked out from the source control repository +implied by its import path. This can be useful if the source is a local fork +of the original. + The -fix flag instructs get to run the fix tool on the downloaded packages before resolving dependencies or building the code. diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go index a83cce8f7..2772452dd 100644 --- a/src/cmd/go/generate.go +++ b/src/cmd/go/generate.go @@ -45,7 +45,7 @@ The arguments are space-separated tokens or double-quoted strings passed to the generator as individual arguments when it is run. Quoted strings use Go syntax and are evaluated before execution; a -quoted string appears a single argument to the generator. +quoted string appears as a single argument to the generator. Go generate sets several variables when it runs the generator: -- cgit v1.2.1 From 5223a2c9fbc6a2633a77586dbfec379e584c779e Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Fri, 5 Dec 2014 09:15:38 +0900 Subject: cmd/go: avoid use of bufio.Scanner in generate Scanner can't handle stupid long lines and there are reports of stupid long lines in production. Note the issue isn't long "//go:generate" lines, but any long line in any Go source file. To be fair, if you're going to have a stupid long line it's not a bad bet you'll want to run it through go generate, because it's some embeddable asset that has been machine generated. (One could ask why that generation process didn't add a newline or two, but we should cope anyway.) Rewrite the file scanner in "go generate" so it can handle arbitrarily long lines, and only stores in memory those lines that start "//go:generate". Also: Adjust the documentation to make clear that it does not parse the file. Fixes issue 9143. Fixes issue 9196. LGTM=rsc, dominik.honnef R=rsc, cespare, minux, dominik.honnef CC=golang-codereviews https://codereview.appspot.com/182970043 --- src/cmd/go/doc.go | 21 ++++++++++----- src/cmd/go/generate.go | 71 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 22 deletions(-) (limited to 'src/cmd/go') diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 879fc7f8b..65640fb48 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -234,17 +234,24 @@ create or update Go source files, for instance by running yacc. Go generate is never run automatically by go build, go get, go test, and so on. It must be run explicitly. -Directives are written as a whole-line comment of the form +Go generate scans the file for directives, which are lines of +the form, //go:generate command argument... -(note: no space in "//go") where command is the generator to be -run, corresponding to an executable file that can be run locally. -It must either be in the shell path (gofmt), a fully qualified path -(/usr/you/bin/mytool), or a command alias, described below. +(note: no leading spaces and no space in "//go") where command +is the generator to be run, corresponding to an executable file +that can be run locally. It must either be in the shell path +(gofmt), a fully qualified path (/usr/you/bin/mytool), or a +command alias, described below. -The arguments are space-separated tokens or double-quoted strings -passed to the generator as individual arguments when it is run. +Note that go generate does not parse the file, so lines that look +like directives in comments or multiline strings will be treated +as directives. + +The arguments to the directive are space-separated tokens or +double-quoted strings passed to the generator as individual +arguments when it is run. Quoted strings use Go syntax and are evaluated before execution; a quoted string appears as a single argument to the generator. diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go index 2772452dd..88f7efa0f 100644 --- a/src/cmd/go/generate.go +++ b/src/cmd/go/generate.go @@ -32,17 +32,24 @@ create or update Go source files, for instance by running yacc. Go generate is never run automatically by go build, go get, go test, and so on. It must be run explicitly. -Directives are written as a whole-line comment of the form +Go generate scans the file for directives, which are lines of +the form, //go:generate command argument... -(note: no space in "//go") where command is the generator to be -run, corresponding to an executable file that can be run locally. -It must either be in the shell path (gofmt), a fully qualified path -(/usr/you/bin/mytool), or a command alias, described below. +(note: no leading spaces and no space in "//go") where command +is the generator to be run, corresponding to an executable file +that can be run locally. It must either be in the shell path +(gofmt), a fully qualified path (/usr/you/bin/mytool), or a +command alias, described below. -The arguments are space-separated tokens or double-quoted strings -passed to the generator as individual arguments when it is run. +Note that go generate does not parse the file, so lines that look +like directives in comments or multiline strings will be treated +as directives. + +The arguments to the directive are space-separated tokens or +double-quoted strings passed to the generator as individual +arguments when it is run. Quoted strings use Go syntax and are evaluated before execution; a quoted string appears as a single argument to the generator. @@ -178,13 +185,43 @@ func (g *Generator) run() (ok bool) { fmt.Fprintf(os.Stderr, "%s\n", shortPath(g.path)) } - s := bufio.NewScanner(g.r) - for s.Scan() { - g.lineNum++ - if !bytes.HasPrefix(s.Bytes(), []byte("//go:generate ")) && !bytes.HasPrefix(s.Bytes(), []byte("//go:generate\t")) { + // Scan for lines that start "//go:generate". + // Can't use bufio.Scanner because it can't handle long lines, + // which are likely to appear when using generate. + input := bufio.NewReader(g.r) + var err error + // One line per loop. + for { + g.lineNum++ // 1-indexed. + var buf []byte + buf, err = input.ReadSlice('\n') + if err == bufio.ErrBufferFull { + // Line too long - consume and ignore. + if isGoGenerate(buf) { + g.errorf("directive too long") + } + for err == bufio.ErrBufferFull { + _, err = input.ReadSlice('\n') + } + if err != nil { + break + } + continue + } + + if err != nil { + // Check for marker at EOF without final \n. + if err == io.EOF && isGoGenerate(buf) { + err = io.ErrUnexpectedEOF + } + break + } + + if !isGoGenerate(buf) { continue } - words := g.split(s.Text()) + + words := g.split(string(buf)) if len(words) == 0 { g.errorf("no arguments to directive") } @@ -201,19 +238,23 @@ func (g *Generator) run() (ok bool) { } g.exec(words) } - if s.Err() != nil { - g.errorf("error reading %s: %s", shortPath(g.path), s.Err()) + if err != nil && err != io.EOF { + g.errorf("error reading %s: %s", shortPath(g.path), err) } return true } +func isGoGenerate(buf []byte) bool { + return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t")) +} + // split breaks the line into words, evaluating quoted // strings and evaluating environment variables. // The initial //go:generate element is dropped. func (g *Generator) split(line string) []string { // Parse line, obeying quoted strings. var words []string - line = line[len("//go:generate "):] + line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline. // One (possibly quoted) word per iteration. Words: for { -- cgit v1.2.1 From 3ebebda3a7495402239db4369d59d73749c1bfa2 Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Fri, 5 Dec 2014 09:37:56 +0900 Subject: cmd/go: fix build The new semantics of split require the newline be present. The test was stale. LGTM=adg R=golang-codereviews, adg CC=golang-codereviews https://codereview.appspot.com/182480043 --- src/cmd/go/generate.go | 2 +- src/cmd/go/generate_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/cmd/go') diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go index 88f7efa0f..baf4d2b55 100644 --- a/src/cmd/go/generate.go +++ b/src/cmd/go/generate.go @@ -250,7 +250,7 @@ func isGoGenerate(buf []byte) bool { // split breaks the line into words, evaluating quoted // strings and evaluating environment variables. -// The initial //go:generate element is dropped. +// The initial //go:generate element is present in line. func (g *Generator) split(line string) []string { // Parse line, obeying quoted strings. var words []string diff --git a/src/cmd/go/generate_test.go b/src/cmd/go/generate_test.go index 93c0ae66e..660ebabbe 100644 --- a/src/cmd/go/generate_test.go +++ b/src/cmd/go/generate_test.go @@ -40,7 +40,7 @@ func TestGenerateCommandParse(t *testing.T) { } g.setShorthand([]string{"-command", "yacc", "go", "tool", "yacc"}) for _, test := range splitTests { - got := g.split("//go:generate " + test.in) + got := g.split("//go:generate " + test.in + "\n") if !reflect.DeepEqual(got, test.out) { t.Errorf("split(%q): got %q expected %q", test.in, got, test.out) } -- cgit v1.2.1