diff options
| author | bstarynk <bstarynk@138bc75d-0d04-0410-961f-82ee72b054a4> | 2016-04-15 06:34:27 +0000 |
|---|---|---|
| committer | bstarynk <bstarynk@138bc75d-0d04-0410-961f-82ee72b054a4> | 2016-04-15 06:34:27 +0000 |
| commit | 058672f818da2e2fd3eed44737939291cc02d809 (patch) | |
| tree | cd6e318f2abeabc01493804644c205c72f5eb7f3 /libgo/go/cmd | |
| parent | ac2c6188d9bb781b7055c102c0af536fd6ca3366 (diff) | |
| download | gcc-058672f818da2e2fd3eed44737939291cc02d809.tar.gz | |
2016-04-15 Basile Starynkevitch <basile@starynkevitch.net>
{{merging with even more of GCC 6, using subversion 1.9
svn merge -r229501:229700 ^/trunk
}}
git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/branches/melt-branch@235005 138bc75d-0d04-0410-961f-82ee72b054a4
Diffstat (limited to 'libgo/go/cmd')
73 files changed, 8020 insertions, 2583 deletions
diff --git a/libgo/go/cmd/cgo/ast.go b/libgo/go/cmd/cgo/ast.go index 10e2278a1d6..8bbd1cc52e6 100644 --- a/libgo/go/cmd/cgo/ast.go +++ b/libgo/go/cmd/cgo/ast.go @@ -235,9 +235,17 @@ func (f *File) saveExport(x interface{}, context string) { error_(c.Pos(), "export comment has wrong name %q, want %q", name, n.Name.Name) } + doc := "" + for _, c1 := range n.Doc.List { + if c1 != c { + doc += c1.Text + "\n" + } + } + f.ExpFunc = append(f.ExpFunc, &ExpFunc{ Func: n, ExpName: name, + Doc: doc, }) break } diff --git a/libgo/go/cmd/cgo/doc.go b/libgo/go/cmd/cgo/doc.go index 6179c7afd19..b2a5428f3f4 100644 --- a/libgo/go/cmd/cgo/doc.go +++ b/libgo/go/cmd/cgo/doc.go @@ -20,16 +20,23 @@ the C parts of the package. For example: // #include <errno.h> import "C" +The preamble may contain any C code, including function and variable +declarations and definitions. These may then be referred to from Go +code as though they were defined in the package "C". All names +declared in the preamble may be used, even if they start with a +lower-case letter. Exception: static variables in the preamble may +not be referenced from Go code; static functions are permitted. + See $GOROOT/misc/cgo/stdio and $GOROOT/misc/cgo/gmp for examples. See "C? Go? Cgo!" for an introduction to using cgo: -http://golang.org/doc/articles/c_go_cgo.html. +https://golang.org/doc/articles/c_go_cgo.html. CFLAGS, CPPFLAGS, CXXFLAGS and LDFLAGS may be defined with pseudo #cgo directives within these comments to tweak the behavior of the C or C++ compiler. Values defined in multiple directives are concatenated together. The directive can include a list of build constraints limiting its effect to systems satisfying one of the constraints -(see http://golang.org/pkg/go/build/#hdr-Build_Constraints for details about the constraint syntax). +(see https://golang.org/pkg/go/build/#hdr-Build_Constraints for details about the constraint syntax). For example: // #cgo CFLAGS: -DPNG_DEBUG=1 @@ -60,6 +67,18 @@ concatenated and used at link time. All the pkg-config directives are concatenated and sent to pkg-config simultaneously to add to each appropriate set of command-line flags. +When the cgo directives are parsed, any occurrence of the string ${SRCDIR} +will be replaced by the absolute path to the directory containing the source +file. This allows pre-compiled static libraries to be included in the package +directory and linked properly. +For example if package foo is in the directory /go/src/foo: + + // #cgo LDFLAGS: -L${SRCDIR}/libs -lfoo + +Will be expanded to: + + // #cgo LDFLAGS: -L/go/src/foo/libs -lfoo + When the Go tool sees that one or more Go files use the special import "C", it will look for other non-Go files in the directory and compile them as part of the Go package. Any .c, .s, or .S files will be @@ -71,17 +90,19 @@ compilers may be changed by the CC and CXX environment variables, respectively; those environment variables may include command line options. -To enable cgo during cross compiling builds, set the CGO_ENABLED -environment variable to 1 when building the Go tools with make.bash. -Also, set CC_FOR_TARGET to the C cross compiler for the target. CC will -be used for compiling for the host. - -After the Go tools are built, when running the go command, CC_FOR_TARGET is -ignored. The value of CC_FOR_TARGET when running make.bash is the default -compiler. However, you can set the environment variable CC, not CC_FOR_TARGET, -to control the compiler when running the go tool. +The cgo tool is enabled by default for native builds on systems where +it is expected to work. It is disabled by default when +cross-compiling. You can control this by setting the CGO_ENABLED +environment variable when running the go tool: set it to 1 to enable +the use of cgo, and to 0 to disable it. The go tool will set the +build constraint "cgo" if cgo is enabled. -CXX_FOR_TARGET works in a similar way for C++ code. +When cross-compiling, you must specify a C cross-compiler for cgo to +use. You can do this by setting the CC_FOR_TARGET environment +variable when building the toolchain using make.bash, or by setting +the CC environment variable any time you run the go tool. The +CXX_FOR_TARGET and CXX environment variables work in a similar way for +C++ code. Go references to C @@ -195,16 +216,18 @@ Not all Go types can be mapped to C types in a useful way. Using //export in a file places a restriction on the preamble: since it is copied into two different C output files, it must not -contain any definitions, only declarations. Definitions must be -placed in preambles in other files, or in C source files. +contain any definitions, only declarations. If a file contains both +definitions and declarations, then the two output files will produce +duplicate symbols and the linker will fail. To avoid this, definitions +must be placed in preambles in other files, or in C source files. Using cgo directly Usage: - go tool cgo [cgo options] [-- compiler options] file.go + go tool cgo [cgo options] [-- compiler options] gofiles... -Cgo transforms the input file.go into four output files: two Go source -files, a C file for 6c (or 8c or 5c), and a C file for gcc. +Cgo transforms the specified input Go source files into several output +Go and C source files. The compiler options are passed through uninterpreted when invoking the C compiler to compile the C parts of the package. @@ -217,18 +240,23 @@ The following options are available when running cgo directly: build when building a cgo package. -dynout file Write -dynimport output to file. + -dynpackage package + Set Go package for -dynimport output. -dynlinker Write dynamic linker as part of -dynimport output. -godefs Write out input file in Go syntax replacing C package names with real values. Used to generate files in the syscall package when bootstrapping a new target. - -cdefs - Like -godefs, but write file in C syntax. - Used to generate files in the runtime package when - bootstrapping a new target. -objdir directory Put all generated files in directory. + -importpath string + The import path for the Go package. Optional; used for + nicer comments in the generated files. + -exportheader file + If there are any exported functions, write the + generated export declarations to file. + C code can #include this to see the declarations. -gccgo Generate output for the gccgo compiler rather than the gc compiler. @@ -363,9 +391,9 @@ the translation process. Translating Go -[The rest of this comment refers to 6g and 6c, the Go and C compilers -that are part of the amd64 port of the gc Go toolchain. Everything here -applies to another architecture's compilers as well.] +[The rest of this comment refers to 6g, the Go compiler that is part +of the amd64 port of the gc Go toolchain. Everything here applies to +another architecture's compilers as well.] Given the input Go files x.go and y.go, cgo generates these source files: @@ -373,44 +401,41 @@ files: x.cgo1.go # for 6g y.cgo1.go # for 6g _cgo_gotypes.go # for 6g - _cgo_defun.c # for 6c + _cgo_import.go # for 6g (if -dynout _cgo_import.go) x.cgo2.c # for gcc y.cgo2.c # for gcc + _cgo_defun.c # for gcc (if -gccgo) _cgo_export.c # for gcc + _cgo_export.h # for gcc _cgo_main.c # for gcc + _cgo_flags # for alternative build tools The file x.cgo1.go is a copy of x.go with the import "C" removed and references to C.xxx replaced with names like _Cfunc_xxx or _Ctype_xxx. The definitions of those identifiers, written as Go functions, types, or variables, are provided in _cgo_gotypes.go. -Here is a _cgo_gotypes.go containing definitions for C.flush (provided -in the preamble) and C.puts (from stdio): +Here is a _cgo_gotypes.go containing definitions for needed C types: type _Ctype_char int8 type _Ctype_int int32 type _Ctype_void [0]byte - func _Cfunc_CString(string) *_Ctype_char - func _Cfunc_flush() _Ctype_void - func _Cfunc_puts(*_Ctype_char) _Ctype_int - -For functions, cgo only writes an external declaration in the Go -output. The implementation is in a combination of C for 6c (meaning -any gc-toolchain compiler) and C for gcc. - -The 6c file contains the definitions of the functions. They all have -similar bodies that invoke runtime·cgocall to make a switch from the -Go runtime world to the system C (GCC-based) world. +The _cgo_gotypes.go file also contains the definitions of the +functions. They all have similar bodies that invoke runtime·cgocall +to make a switch from the Go runtime world to the system C (GCC-based) +world. For example, here is the definition of _Cfunc_puts: - void _cgo_be59f0f25121_Cfunc_puts(void*); + //go:cgo_import_static _cgo_be59f0f25121_Cfunc_puts + //go:linkname __cgofn__cgo_be59f0f25121_Cfunc_puts _cgo_be59f0f25121_Cfunc_puts + var __cgofn__cgo_be59f0f25121_Cfunc_puts byte + var _cgo_be59f0f25121_Cfunc_puts = unsafe.Pointer(&__cgofn__cgo_be59f0f25121_Cfunc_puts) - void - ·_Cfunc_puts(struct{uint8 x[1];}p) - { - runtime·cgocall(_cgo_be59f0f25121_Cfunc_puts, &p); + func _Cfunc_puts(p0 *_Ctype_char) (r1 _Ctype_int) { + _cgo_runtime_cgocall(_cgo_be59f0f25121_Cfunc_puts, uintptr(unsafe.Pointer(&p0))) + return } The hexadecimal number is a hash of cgo's input, chosen to be @@ -421,6 +446,7 @@ file compiled by gcc, the file x.cgo2.c: void _cgo_be59f0f25121_Cfunc_puts(void *v) { + _cgo_wait_runtime_init_done(); struct { char* p0; int r; @@ -429,7 +455,8 @@ file compiled by gcc, the file x.cgo2.c: a->r = puts((void*)a->p0); } -It extracts the arguments from the pointer to _Cfunc_puts's argument +It waits for Go runtime to be initialized (required for shared libraries), +extracts the arguments from the pointer to _Cfunc_puts's argument frame, invokes the system C function (in this case, puts), stores the result in the frame, and returns. @@ -448,6 +475,7 @@ _cgo_main.c: int main() { return 0; } void crosscall2(void(*fn)(void*, int), void *a, int c) { } + void _cgo_wait_runtime_init_done() { } void _cgo_allocate(void *a, int c) { } void _cgo_panic(void *a, int c) { } @@ -456,23 +484,21 @@ code generated for gcc. The build process links this stub, along with _cgo_export.c and *.cgo2.c, into a dynamic executable and then lets cgo examine the executable. Cgo records the list of shared library references and resolved names and writes them into a new file -_cgo_import.c, which looks like: +_cgo_import.go, which looks like: - #pragma cgo_dynamic_linker "/lib64/ld-linux-x86-64.so.2" - #pragma cgo_import_dynamic puts puts#GLIBC_2.2.5 "libc.so.6" - #pragma cgo_import_dynamic __libc_start_main __libc_start_main#GLIBC_2.2.5 "libc.so.6" - #pragma cgo_import_dynamic stdout stdout#GLIBC_2.2.5 "libc.so.6" - #pragma cgo_import_dynamic fflush fflush#GLIBC_2.2.5 "libc.so.6" - #pragma cgo_import_dynamic _ _ "libpthread.so.0" - #pragma cgo_import_dynamic _ _ "libc.so.6" + //go:cgo_dynamic_linker "/lib64/ld-linux-x86-64.so.2" + //go:cgo_import_dynamic puts puts#GLIBC_2.2.5 "libc.so.6" + //go:cgo_import_dynamic __libc_start_main __libc_start_main#GLIBC_2.2.5 "libc.so.6" + //go:cgo_import_dynamic stdout stdout#GLIBC_2.2.5 "libc.so.6" + //go:cgo_import_dynamic fflush fflush#GLIBC_2.2.5 "libc.so.6" + //go:cgo_import_dynamic _ _ "libpthread.so.0" + //go:cgo_import_dynamic _ _ "libc.so.6" In the end, the compiled Go package, which will eventually be presented to 6l as part of a larger program, contains: - _go_.6 # 6g-compiled object for _cgo_gotypes.go *.cgo1.go - _cgo_defun.6 # 6c-compiled object for _cgo_defun.c + _go_.6 # 6g-compiled object for _cgo_gotypes.go, _cgo_import.go, *.cgo1.go _all.o # gcc-compiled object for _cgo_export.c, *.cgo2.c - _cgo_import.6 # 6c-compiled object for _cgo_import.c The final program will be a dynamic executable, so that 6l can avoid needing to process arbitrary .o files. It only needs to process the .o @@ -496,20 +522,21 @@ Runtime When using cgo, Go must not assume that it owns all details of the process. In particular it needs to coordinate with C in the use of -threads and thread-local storage. The runtime package, in its own -(6c-compiled) C code, declares a few uninitialized (default bss) +threads and thread-local storage. The runtime package declares a few variables: - bool runtime·iscgo; - void (*libcgo_thread_start)(void*); - void (*initcgo)(G*); + var ( + iscgo bool + _cgo_init unsafe.Pointer + _cgo_thread_start unsafe.Pointer + ) Any package using cgo imports "runtime/cgo", which provides -initializations for these variables. It sets iscgo to 1, initcgo to a -gcc-compiled function that can be called early during program startup, -and libcgo_thread_start to a gcc-compiled function that can be used to -create a new thread, in place of the runtime's usual direct system -calls. +initializations for these variables. It sets iscgo to true, _cgo_init +to a gcc-compiled function that can be called early during program +startup, and _cgo_thread_start to a gcc-compiled function that can be +used to create a new thread, in place of the runtime's usual direct +system calls. Internal and External Linking @@ -522,12 +549,12 @@ code can only be used as a dynamic library). On the other hand, when using internal linking, 6l can generate Go binaries by itself. In order to allow linking arbitrary object files without requiring -dynamic libraries, cgo will soon support an "external" linking mode -too. In external linking mode, 6l does not process any host object -files. Instead, it collects all the Go code and writes a single go.o -object file containing it. Then it invokes the host linker (usually -gcc) to combine the go.o object file and any supporting non-Go code -into a final executable. External linking avoids the dynamic library +dynamic libraries, cgo supports an "external" linking mode too. In +external linking mode, 6l does not process any host object files. +Instead, it collects all the Go code and writes a single go.o object +file containing it. Then it invokes the host linker (usually gcc) to +combine the go.o object file and any supporting non-Go code into a +final executable. External linking avoids the dynamic library requirement but introduces a requirement that the host linker be present to create such a binary. @@ -555,13 +582,13 @@ to be made when linking the final binary. Linking Directives In either linking mode, package-specific directives must be passed -through to 6l. These are communicated by writing #pragma directives -in a C source file compiled by 6c. The directives are copied into the .6 object file -and then processed by the linker. +through to 6l. These are communicated by writing //go: directives in a +Go source file compiled by 6g. The directives are copied into the .6 +object file and then processed by the linker. The directives are: -#pragma cgo_import_dynamic <local> [<remote> ["<library>"]] +//go:cgo_import_dynamic <local> [<remote> ["<library>"]] In internal linking mode, allow an unresolved reference to <local>, assuming it will be resolved by a dynamic library @@ -572,9 +599,9 @@ The directives are: In the <remote>, # or @ can be used to introduce a symbol version. Examples: - #pragma cgo_import_dynamic puts - #pragma cgo_import_dynamic puts puts#GLIBC_2.2.5 - #pragma cgo_import_dynamic puts puts#GLIBC_2.2.5 "libc.so.6" + //go:cgo_import_dynamic puts + //go:cgo_import_dynamic puts puts#GLIBC_2.2.5 + //go:cgo_import_dynamic puts puts#GLIBC_2.2.5 "libc.so.6" A side effect of the cgo_import_dynamic directive with a library is to make the final binary depend on that dynamic @@ -582,12 +609,12 @@ The directives are: symbols, use _ for local and remote. Example: - #pragma cgo_import_dynamic _ _ "libc.so.6" + //go:cgo_import_dynamic _ _ "libc.so.6" For compatibility with current versions of SWIG, - #pragma dynimport is an alias for #pragma cgo_import_dynamic. + #pragma dynimport is an alias for //go:cgo_import_dynamic. -#pragma cgo_dynamic_linker "<path>" +//go:cgo_dynamic_linker "<path>" In internal linking mode, use "<path>" as the dynamic linker in the final binary. This directive is only needed from one @@ -595,9 +622,9 @@ The directives are: supplied by runtime/cgo. Example: - #pragma cgo_dynamic_linker "/lib/ld-linux.so.2" + //go:cgo_dynamic_linker "/lib/ld-linux.so.2" -#pragma cgo_export_dynamic <local> <remote> +//go:cgo_export_dynamic <local> <remote> In internal linking mode, put the Go symbol named <local> into the program's exported symbol table as @@ -606,9 +633,9 @@ The directives are: to share Go's data. For compatibility with current versions of SWIG, - #pragma dynexport is an alias for #pragma cgo_export_dynamic. + #pragma dynexport is an alias for //go:cgo_export_dynamic. -#pragma cgo_import_static <local> +//go:cgo_import_static <local> In external linking mode, allow unresolved references to <local> in the go.o object file prepared for the host linker, @@ -616,9 +643,9 @@ The directives are: other object files that will be linked with go.o. Example: - #pragma cgo_import_static puts_wrapper + //go:cgo_import_static puts_wrapper -#pragma cgo_export_static <local> <remote> +//go:cgo_export_static <local> <remote> In external linking mode, put the Go symbol named <local> into the program's exported symbol table as @@ -626,15 +653,15 @@ The directives are: mechanism makes it possible for C code to call back into Go or to share Go's data. -#pragma cgo_ldflag "<arg>" +//go:cgo_ldflag "<arg>" In external linking mode, invoke the host linker (usually gcc) with "<arg>" as a command-line argument following the .o files. Note that the arguments are for "gcc", not "ld". Example: - #pragma cgo_ldflag "-lpthread" - #pragma cgo_ldflag "-L/usr/local/sqlite3/lib" + //go:cgo_ldflag "-lpthread" + //go:cgo_ldflag "-L/usr/local/sqlite3/lib" A package compiled with cgo will include directives for both internal and external linking; the linker will select the appropriate @@ -647,22 +674,18 @@ The following code will be generated by cgo: // compiled by 6g - type _Ctype_double float64 - func _Cfunc_sin(_Ctype_double) _Ctype_double - - // compiled by 6c + //go:cgo_ldflag "-lm" - #pragma cgo_import_dynamic sin sin#GLIBC_2.2.5 "libm.so.6" - - #pragma cgo_import_static _cgo_gcc_Cfunc_sin - #pragma cgo_ldflag "-lm" + type _Ctype_double float64 - void _cgo_gcc_Cfunc_sin(void*); + //go:cgo_import_static _cgo_gcc_Cfunc_sin + //go:linkname __cgo_gcc_Cfunc_sin _cgo_gcc_Cfunc_sin + var __cgo_gcc_Cfunc_sin byte + var _cgo_gcc_Cfunc_sin = unsafe.Pointer(&__cgo_gcc_Cfunc_sin) - void - ·_Cfunc_sin(struct{uint8 x[16];}p) - { - runtime·cgocall(_cgo_gcc_Cfunc_sin, &p); + func _Cfunc_sin(p0 _Ctype_double) (r1 _Ctype_double) { + _cgo_runtime_cgocall(_cgo_gcc_Cfunc_sin, uintptr(unsafe.Pointer(&p0))) + return } // compiled by gcc, into foo.cgo2.o @@ -682,8 +705,8 @@ using the internal or external mode. If other packages are compiled in "external only" mode, then the final link will be an external one. Otherwise the link will be an internal one. -The directives in the 6c-compiled file are used according to the kind -of final link used. +The linking directives are used according to the kind of final link +used. In internal mode, 6l itself processes all the host object files, in particular foo.cgo2.o. To do so, it uses the cgo_import_dynamic and @@ -694,10 +717,10 @@ symbol sin with version GLIBC_2.2.5 from the dynamic library runtime dynamic linker. In external mode, 6l does not process any host object files, in -particular foo.cgo2.o. It links together the 6g- and 6c-generated -object files, along with any other Go code, into a go.o file. While -doing that, 6l will discover that there is no definition for -_cgo_gcc_Cfunc_sin, referred to by the 6c-compiled source file. This +particular foo.cgo2.o. It links together the 6g-generated object +files, along with any other Go code, into a go.o file. While doing +that, 6l will discover that there is no definition for +_cgo_gcc_Cfunc_sin, referred to by the 6g-compiled source file. This is okay, because 6l also processes the cgo_import_static directive and knows that _cgo_gcc_Cfunc_sin is expected to be supplied by a host object file, so 6l does not treat the missing symbol as an error when diff --git a/libgo/go/cmd/cgo/gcc.go b/libgo/go/cmd/cgo/gcc.go index abdd369d713..b64849a8d16 100644 --- a/libgo/go/cmd/cgo/gcc.go +++ b/libgo/go/cmd/cgo/gcc.go @@ -154,20 +154,6 @@ func splitQuoted(s string) (r []string, err error) { return args, err } -var safeBytes = []byte(`+-.,/0123456789:=ABCDEFGHIJKLMNOPQRSTUVWXYZ\_abcdefghijklmnopqrstuvwxyz`) - -func safeName(s string) bool { - if s == "" { - return false - } - for i := 0; i < len(s); i++ { - if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { - return false - } - } - return true -} - // Translate rewrites f.AST, the original Go input, to remove // references to the imported package C, replacing them with // references to the equivalent Go types, functions, and variables. @@ -213,6 +199,10 @@ func (p *Package) loadDefines(f *File) { val = strings.TrimSpace(line[tabIndex:]) } + if key == "__clang__" { + p.GccIsClang = true + } + if n := f.Name[key]; n != nil { if *debugDefine { fmt.Fprintf(os.Stderr, "#define %s %s\n", key, val) @@ -582,7 +572,7 @@ func (p *Package) mangleName(n *Name) { // rewriteRef rewrites all the C.xxx references in f.AST to refer to the // Go equivalents, now that we have figured out the meaning of all -// the xxx. In *godefs or *cdefs mode, rewriteRef replaces the names +// the xxx. In *godefs mode, rewriteRef replaces the names // with full definitions instead of mangled names. func (p *Package) rewriteRef(f *File) { // Keep a list of all the functions, to remove the ones @@ -673,6 +663,13 @@ func (p *Package) rewriteRef(f *File) { expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr} } + case "selector": + if r.Name.Kind == "var" { + expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr} + } else { + error_(r.Pos(), "only C variables allowed in selector expression", fixGo(r.Name.Go)) + } + case "type": if r.Name.Kind != "type" { error_(r.Pos(), "expression C.%s used as type", fixGo(r.Name.Go)) @@ -688,7 +685,7 @@ func (p *Package) rewriteRef(f *File) { error_(r.Pos(), "must call C.%s", fixGo(r.Name.Go)) } } - if *godefs || *cdefs { + if *godefs { // Substitute definition for mangled type name. if id, ok := expr.(*ast.Ident); ok { if t := typedef[id.Name]; t != nil { @@ -746,6 +743,10 @@ func (p *Package) gccMachine() []string { return []string{"-m32"} case "arm": return []string{"-marm"} // not thumb + case "s390": + return []string{"-m31"} + case "s390x": + return []string{"-m64"} } return nil } @@ -765,7 +766,7 @@ func (p *Package) gccCmd() []string { "-c", // do not link "-xc", // input language is C ) - if strings.Contains(c[0], "clang") { + if p.GccIsClang { c = append(c, "-ferror-limit=0", // Apple clang version 1.7 (tags/Apple/clang-77) (based on LLVM 2.9svn) @@ -780,7 +781,7 @@ func (p *Package) gccCmd() []string { // incorrectly typed unsigned long. We work around that // by disabling the builtin functions (this is safe as // it won't affect the actual compilation of the C code). - // See: http://golang.org/issue/6506. + // See: https://golang.org/issue/6506. "-fno-builtin", ) } @@ -992,8 +993,8 @@ func (c *typeConv) Init(ptrSize, intSize int64) { c.goVoid = c.Ident("_Ctype_void") // Normally cgo translates void* to unsafe.Pointer, - // but for historical reasons -cdefs and -godefs use *byte instead. - if *cdefs || *godefs { + // but for historical reasons -godefs uses *byte instead. + if *godefs { c.goVoidPtr = &ast.StarExpr{X: c.byte} } else { c.goVoidPtr = c.Ident("unsafe.Pointer") @@ -1045,7 +1046,7 @@ func (tr *TypeRepr) String() string { return fmt.Sprintf(tr.Repr, tr.FormatArgs...) } -// Empty returns true if the result of String would be "". +// Empty reports whether the result of String would be "". func (tr *TypeRepr) Empty() bool { return len(tr.Repr) == 0 } @@ -1334,8 +1335,8 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { // If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo", // use that as the Go form for this typedef too, so that the typedef will be interchangeable // with the base type. - // In -godefs and -cdefs mode, do this for all typedefs. - if isStructUnionClass(sub.Go) || *godefs || *cdefs { + // In -godefs mode, do this for all typedefs. + if isStructUnionClass(sub.Go) || *godefs { t.Go = sub.Go if isStructUnionClass(sub.Go) { @@ -1397,7 +1398,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { name := c.Ident("_Ctype_" + s) tt := *t typedef[name.Name] = &tt - if !*godefs && !*cdefs { + if !*godefs { t.Go = name } } @@ -1543,14 +1544,16 @@ func (c *typeConv) intExpr(n int64) ast.Expr { } // Add padding of given size to fld. -func (c *typeConv) pad(fld []*ast.Field, size int64) []*ast.Field { +func (c *typeConv) pad(fld []*ast.Field, sizes []int64, size int64) ([]*ast.Field, []int64) { n := len(fld) fld = fld[0 : n+1] fld[n] = &ast.Field{Names: []*ast.Ident{c.Ident("_")}, Type: c.Opaque(size)} - return fld + sizes = sizes[0 : n+1] + sizes[n] = size + return fld, sizes } -// Struct conversion: return Go and (6g) C syntax for type. +// Struct conversion: return Go and (gc) C syntax for type. func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.StructType, csyntax string, align int64) { // Minimum alignment for a struct is 1 byte. align = 1 @@ -1558,6 +1561,7 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct var buf bytes.Buffer buf.WriteString("struct {") fld := make([]*ast.Field, 0, 2*len(dt.Field)+1) // enough for padding around every field + sizes := make([]int64, 0, 2*len(dt.Field)+1) off := int64(0) // Rename struct fields that happen to be named Go keywords into @@ -1573,7 +1577,7 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct used[f.Name] = true } - if !*godefs && !*cdefs { + if !*godefs { for cid, goid := range ident { if token.Lookup(goid).IsKeyword() { // Avoid keyword @@ -1593,19 +1597,19 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct anon := 0 for _, f := range dt.Field { if f.ByteOffset > off { - fld = c.pad(fld, f.ByteOffset-off) + fld, sizes = c.pad(fld, sizes, f.ByteOffset-off) off = f.ByteOffset } name := f.Name ft := f.Type - // In godefs or cdefs mode, if this field is a C11 + // In godefs mode, if this field is a C11 // anonymous union then treat the first field in the // union as the field in the struct. This handles // cases like the glibc <sys/resource.h> file; see // issue 6677. - if *godefs || *cdefs { + if *godefs { if st, ok := f.Type.(*dwarf.StructType); ok && name == "" && st.Kind == "union" && len(st.Field) > 0 && !used[st.Field[0].Name] { name = st.Field[0].Name ident[name] = name @@ -1635,14 +1639,12 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct talign = size } - if talign > 0 && f.ByteOffset%talign != 0 && !*cdefs { + if talign > 0 && f.ByteOffset%talign != 0 { // Drop misaligned fields, the same way we drop integer bit fields. // The goal is to make available what can be made available. // Otherwise one bad and unneeded field in an otherwise okay struct // makes the whole program not compile. Much of the time these // structs are in system headers that cannot be corrected. - // Exception: In -cdefs mode, we use #pragma pack, so misaligned - // fields should still work. continue } n := len(fld) @@ -1653,6 +1655,8 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct ident[name] = name } fld[n] = &ast.Field{Names: []*ast.Ident{c.Ident(ident[name])}, Type: tgo} + sizes = sizes[0 : n+1] + sizes[n] = size off += size buf.WriteString(t.C.String()) buf.WriteString(" ") @@ -1663,16 +1667,29 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct } } if off < dt.ByteSize { - fld = c.pad(fld, dt.ByteSize-off) + fld, sizes = c.pad(fld, sizes, dt.ByteSize-off) off = dt.ByteSize } + + // If the last field in a non-zero-sized struct is zero-sized + // the compiler is going to pad it by one (see issue 9401). + // We can't permit that, because then the size of the Go + // struct will not be the same as the size of the C struct. + // Our only option in such a case is to remove the field, + // which means that it can not be referenced from Go. + for off > 0 && sizes[len(sizes)-1] == 0 { + n := len(sizes) + fld = fld[0 : n-1] + sizes = sizes[0 : n-1] + } + if off != dt.ByteSize { fatalf("%s: struct size calculation error off=%d bytesize=%d", lineno(pos), off, dt.ByteSize) } buf.WriteString("}") csyntax = buf.String() - if *godefs || *cdefs { + if *godefs { godefsFields(fld) } expr = &ast.StructType{Fields: &ast.FieldList{List: fld}} @@ -1707,9 +1724,7 @@ func godefsFields(fld []*ast.Field) { n.Name = "Pad_cgo_" + strconv.Itoa(npad) npad++ } - if !*cdefs { - n.Name = upper(n.Name) - } + n.Name = upper(n.Name) } } } @@ -1721,9 +1736,6 @@ func godefsFields(fld []*ast.Field) { // package syscall's data structures, we drop a common prefix // (so sec, usec, which will get turned into Sec, Usec for exporting). func fieldPrefix(fld []*ast.Field) string { - if *cdefs { - return "" - } prefix := "" for _, f := range fld { for _, n := range f.Names { diff --git a/libgo/go/cmd/cgo/godefs.go b/libgo/go/cmd/cgo/godefs.go index ce5ac2736c5..1b0ece29ef4 100644 --- a/libgo/go/cmd/cgo/godefs.go +++ b/libgo/go/cmd/cgo/godefs.go @@ -114,173 +114,6 @@ func (p *Package) godefs(f *File, srcfile string) string { return buf.String() } -// cdefs returns the output for -cdefs mode. -// The easiest way to do this is to translate the godefs Go to C. -func (p *Package) cdefs(f *File, srcfile string) string { - godefsOutput := p.godefs(f, srcfile) - - lines := strings.Split(godefsOutput, "\n") - lines[0] = "// Created by cgo -cdefs - DO NOT EDIT" - - for i, line := range lines { - lines[i] = strings.TrimSpace(line) - } - - var out bytes.Buffer - printf := func(format string, args ...interface{}) { fmt.Fprintf(&out, format, args...) } - - didTypedef := false - for i := 0; i < len(lines); i++ { - line := lines[i] - - // Delete - // package x - if strings.HasPrefix(line, "package ") { - continue - } - - // Convert - // const ( - // A = 1 - // B = 2 - // ) - // - // to - // - // enum { - // A = 1, - // B = 2, - // }; - if line == "const (" { - printf("enum {\n") - for i++; i < len(lines) && lines[i] != ")"; i++ { - line = lines[i] - if line != "" { - printf("\t%s,", line) - } - printf("\n") - } - printf("};\n") - continue - } - - // Convert - // const A = 1 - // to - // enum { A = 1 }; - if strings.HasPrefix(line, "const ") { - printf("enum { %s };\n", line[len("const "):]) - continue - } - - // On first type definition, typedef all the structs - // in case there are dependencies between them. - if !didTypedef && strings.HasPrefix(line, "type ") { - didTypedef = true - for _, line := range lines { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") { - s := strings.TrimSuffix(strings.TrimPrefix(line, "type "), " struct {") - printf("typedef struct %s %s;\n", s, s) - } - } - printf("\n") - printf("#pragma pack on\n") - printf("\n") - } - - // Convert - // type T struct { - // X int64 - // Y *int32 - // Z [4]byte - // } - // - // to - // - // struct T { - // int64 X; - // int32 *Y; - // byte Z[4]; - // } - if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") { - if len(lines) > i+1 && lines[i+1] == "}" { - // do not output empty struct - i++ - continue - } - s := line[len("type ") : len(line)-len(" struct {")] - printf("struct %s {\n", s) - for i++; i < len(lines) && lines[i] != "}"; i++ { - line := lines[i] - if line != "" { - f := strings.Fields(line) - if len(f) != 2 { - fmt.Fprintf(os.Stderr, "cgo: cannot parse struct field: %s\n", line) - nerrors++ - continue - } - printf("\t%s;", cdecl(f[0], f[1])) - } - printf("\n") - } - printf("};\n") - continue - } - - // Convert - // type T int - // to - // typedef int T; - if strings.HasPrefix(line, "type ") { - f := strings.Fields(line[len("type "):]) - if len(f) != 2 { - fmt.Fprintf(os.Stderr, "cgo: cannot parse type definition: %s\n", line) - nerrors++ - continue - } - printf("typedef\t%s;\n", cdecl(f[0], f[1])) - continue - } - - printf("%s\n", line) - } - - if didTypedef { - printf("\n") - printf("#pragma pack off\n") - } - - return out.String() -} - -// cdecl returns the C declaration for the given Go name and type. -// It only handles the specific cases necessary for converting godefs output. -func cdecl(name, typ string) string { - // X *[0]byte -> X *void - if strings.HasPrefix(typ, "*[0]") { - typ = "*void" - } - // X [4]byte -> X[4] byte - for strings.HasPrefix(typ, "[") { - i := strings.Index(typ, "]") + 1 - name = name + typ[:i] - typ = typ[i:] - } - // X *byte -> *X byte - for strings.HasPrefix(typ, "*") { - name = "*" + name - typ = typ[1:] - } - // X T -> T X - // Handle the special case: 'unsafe.Pointer' is 'void *' - if typ == "unsafe.Pointer" { - typ = "void" - name = "*" + name - } - return typ + "\t" + name -} - var gofmtBuf bytes.Buffer // gofmt returns the gofmt-formatted string for an AST node. diff --git a/libgo/go/cmd/cgo/main.go b/libgo/go/cmd/cgo/main.go index 48257fc6c4d..c8cd1610baf 100644 --- a/libgo/go/cmd/cgo/main.go +++ b/libgo/go/cmd/cgo/main.go @@ -6,7 +6,7 @@ // TODO(rsc): // Emit correct line number annotations. -// Make 6g understand the annotations. +// Make gc understand the annotations. package main @@ -33,6 +33,7 @@ type Package struct { PtrSize int64 IntSize int64 GccOptions []string + GccIsClang bool CgoFlags map[string][]string // #cgo flags (CFLAGS, LDFLAGS) Written map[string]bool Name map[string]*Name // accumulated Name from Files @@ -87,7 +88,7 @@ type Name struct { Const string // constant definition } -// IsVar returns true if Kind is either "var" or "fpvar" +// IsVar reports whether Kind is either "var" or "fpvar" func (n *Name) IsVar() bool { return n.Kind == "var" || n.Kind == "fpvar" } @@ -98,6 +99,7 @@ func (n *Name) IsVar() bool { type ExpFunc struct { Func *ast.FuncDecl ExpName string // name to use from C + Doc string } // A TypeRepr contains the string representation of a type. @@ -174,15 +176,18 @@ var cPrefix string var fset = token.NewFileSet() var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file") -var dynout = flag.String("dynout", "", "write -dynobj output to this file") -var dynlinker = flag.Bool("dynlinker", false, "record dynamic linker information in dynimport mode") +var dynout = flag.String("dynout", "", "write -dynimport output to this file") +var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output") +var dynlinker = flag.Bool("dynlinker", false, "record dynamic linker information in -dynimport mode") -// These flags are for bootstrapping a new Go implementation, -// to generate Go and C headers that match the data layout and +// This flag is for bootstrapping a new Go implementation, +// to generate Go types that match the data layout and // constant values used in the host's C libraries and system calls. var godefs = flag.Bool("godefs", false, "for bootstrap: write Go definitions for C file to standard output") -var cdefs = flag.Bool("cdefs", false, "for bootstrap: write C definitions for C file to standard output") + var objDir = flag.String("objdir", "", "object directory") +var importPath = flag.String("importpath", "", "import path of package being built (for comments in generated files)") +var exportHeader = flag.String("exportheader", "", "where to write export header if any exported functions") var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo") var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo") @@ -208,12 +213,7 @@ func main() { return } - if *godefs && *cdefs { - fmt.Fprintf(os.Stderr, "cgo: cannot use -cdefs and -godefs together\n") - os.Exit(2) - } - - if *godefs || *cdefs { + if *godefs { // Generating definitions pulled from header files, // to be checked into Go repositories. // Line numbers are just noise. @@ -305,14 +305,12 @@ func main() { p.Record(f) if *godefs { os.Stdout.WriteString(p.godefs(f, input)) - } else if *cdefs { - os.Stdout.WriteString(p.cdefs(f, input)) } else { p.writeOutput(f, input) } } - if !*godefs && !*cdefs { + if !*godefs { p.writeDefs() } if nerrors > 0 { diff --git a/libgo/go/cmd/cgo/out.go b/libgo/go/cmd/cgo/out.go index d92bed9bf01..90a74419622 100644 --- a/libgo/go/cmd/cgo/out.go +++ b/libgo/go/cmd/cgo/out.go @@ -13,6 +13,7 @@ import ( "go/ast" "go/printer" "go/token" + "io" "os" "sort" "strings" @@ -20,11 +21,17 @@ import ( var conf = printer.Config{Mode: printer.SourcePos, Tabwidth: 8} -// writeDefs creates output files to be compiled by 6g, 6c, and gcc. -// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.) +// writeDefs creates output files to be compiled by gc and gcc. func (p *Package) writeDefs() { - fgo2 := creat(*objDir + "_cgo_gotypes.go") - fc := creat(*objDir + "_cgo_defun.c") + var fgo2, fc io.Writer + f := creat(*objDir + "_cgo_gotypes.go") + defer f.Close() + fgo2 = f + if *gccgo { + f := creat(*objDir + "_cgo_defun.c") + defer f.Close() + fc = f + } fm := creat(*objDir + "_cgo_main.c") var gccgoInit bytes.Buffer @@ -34,7 +41,7 @@ func (p *Package) writeDefs() { fmt.Fprintf(fflg, "_CGO_%s=%s\n", k, strings.Join(v, " ")) if k == "LDFLAGS" && !*gccgo { for _, arg := range v { - fmt.Fprintf(fc, "#pragma cgo_ldflag %q\n", arg) + fmt.Fprintf(fgo2, "//go:cgo_ldflag %q\n", arg) } } } @@ -44,14 +51,17 @@ func (p *Package) writeDefs() { fmt.Fprintf(fm, "int main() { return 0; }\n") if *importRuntimeCgo { fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int), void *a, int c) { }\n") + fmt.Fprintf(fm, "void _cgo_wait_runtime_init_done() { }\n") fmt.Fprintf(fm, "char* _cgo_topofstack(void) { return (char*)0; }\n") } else { // If we're not importing runtime/cgo, we *are* runtime/cgo, - // which provides crosscall2. We just need a prototype. + // which provides these functions. We just need a prototype. fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int), void *a, int c);\n") + fmt.Fprintf(fm, "void _cgo_wait_runtime_init_done();\n") } fmt.Fprintf(fm, "void _cgo_allocate(void *a, int c) { }\n") fmt.Fprintf(fm, "void _cgo_panic(void *a, int c) { }\n") + fmt.Fprintf(fm, "void _cgo_reginit(void) { }\n") // Write second Go output: definitions of _C_xxx. // In a separate file so that the import of "unsafe" does not @@ -68,6 +78,13 @@ func (p *Package) writeDefs() { } fmt.Fprintf(fgo2, "func _Cgo_ptr(ptr unsafe.Pointer) unsafe.Pointer { return ptr }\n\n") + if !*gccgo { + fmt.Fprintf(fgo2, "//go:linkname _Cgo_always_false runtime.cgoAlwaysFalse\n") + fmt.Fprintf(fgo2, "var _Cgo_always_false bool\n") + fmt.Fprintf(fgo2, "//go:linkname _Cgo_use runtime.cgoUse\n") + fmt.Fprintf(fgo2, "func _Cgo_use(interface{})\n") + } + typedefNames := make([]string, 0, len(typedef)) for name := range typedef { typedefNames = append(typedefNames, name) @@ -88,7 +105,6 @@ func (p *Package) writeDefs() { if *gccgo { fmt.Fprint(fc, p.cPrologGccgo()) } else { - fmt.Fprint(fc, cProlog) fmt.Fprint(fgo2, goProlog) } @@ -102,44 +118,44 @@ func (p *Package) writeDefs() { } if !cVars[n.C] { - fmt.Fprintf(fm, "extern char %s[];\n", n.C) - fmt.Fprintf(fm, "void *_cgohack_%s = %s;\n\n", n.C, n.C) - - if !*gccgo { - fmt.Fprintf(fc, "#pragma cgo_import_static %s\n", n.C) + if *gccgo { + fmt.Fprintf(fc, "extern byte *%s;\n", n.C) + } else { + fmt.Fprintf(fm, "extern char %s[];\n", n.C) + fmt.Fprintf(fm, "void *_cgohack_%s = %s;\n\n", n.C, n.C) + fmt.Fprintf(fgo2, "//go:linkname __cgo_%s %s\n", n.C, n.C) + fmt.Fprintf(fgo2, "//go:cgo_import_static %s\n", n.C) + fmt.Fprintf(fgo2, "var __cgo_%s byte\n", n.C) } - - fmt.Fprintf(fc, "extern byte *%s;\n", n.C) - cVars[n.C] = true } - var amp string + var node ast.Node if n.Kind == "var" { - amp = "&" node = &ast.StarExpr{X: n.Type.Go} } else if n.Kind == "fpvar" { node = n.Type.Go - if *gccgo { - amp = "&" - } } else { panic(fmt.Errorf("invalid var kind %q", n.Kind)) } if *gccgo { fmt.Fprintf(fc, `extern void *%s __asm__("%s.%s");`, n.Mangle, gccgoSymbolPrefix, n.Mangle) - fmt.Fprintf(&gccgoInit, "\t%s = %s%s;\n", n.Mangle, amp, n.C) - } else { - fmt.Fprintf(fc, "#pragma dataflag NOPTR /* C pointer, not heap pointer */ \n") - fmt.Fprintf(fc, "void *·%s = %s%s;\n", n.Mangle, amp, n.C) + fmt.Fprintf(&gccgoInit, "\t%s = &%s;\n", n.Mangle, n.C) + fmt.Fprintf(fc, "\n") } - fmt.Fprintf(fc, "\n") fmt.Fprintf(fgo2, "var %s ", n.Mangle) conf.Fprint(fgo2, fset, node) + if !*gccgo { + fmt.Fprintf(fgo2, " = (") + conf.Fprint(fgo2, fset, node) + fmt.Fprintf(fgo2, ")(unsafe.Pointer(&__cgo_%s))", n.C) + } fmt.Fprintf(fgo2, "\n") } - fmt.Fprintf(fc, "\n") + if *gccgo { + fmt.Fprintf(fc, "\n") + } for _, key := range nameKeys(p.Name) { n := p.Name[key] @@ -152,14 +168,37 @@ func (p *Package) writeDefs() { for _, key := range nameKeys(p.Name) { n := p.Name[key] if n.FuncType != nil { - p.writeDefsFunc(fc, fgo2, n) + p.writeDefsFunc(fgo2, n) } } + fgcc := creat(*objDir + "_cgo_export.c") + fgcch := creat(*objDir + "_cgo_export.h") if *gccgo { - p.writeGccgoExports(fgo2, fc, fm) + p.writeGccgoExports(fgo2, fm, fgcc, fgcch) } else { - p.writeExports(fgo2, fc, fm) + p.writeExports(fgo2, fm, fgcc, fgcch) + } + if err := fgcc.Close(); err != nil { + fatalf("%s", err) + } + if err := fgcch.Close(); err != nil { + fatalf("%s", err) + } + + if *exportHeader != "" && len(p.ExpFunc) > 0 { + fexp := creat(*exportHeader) + fgcch, err := os.Open(*objDir + "_cgo_export.h") + if err != nil { + fatalf("%s", err) + } + _, err = io.Copy(fexp, fgcch) + if err != nil { + fatalf("%s", err) + } + if err = fexp.Close(); err != nil { + fatalf("%s", err) + } } init := gccgoInit.String() @@ -169,9 +208,6 @@ func (p *Package) writeDefs() { fmt.Fprint(fc, init) fmt.Fprintln(fc, "}") } - - fgo2.Close() - fc.Close() } func dynimport(obj string) { @@ -184,13 +220,15 @@ func dynimport(obj string) { stdout = f } + fmt.Fprintf(stdout, "package %s\n", *dynpackage) + if f, err := elf.Open(obj); err == nil { if *dynlinker { // Emit the cgo_dynamic_linker line. if sec := f.Section(".interp"); sec != nil { if data, err := sec.Data(); err == nil && len(data) > 1 { // skip trailing \0 in data - fmt.Fprintf(stdout, "#pragma cgo_dynamic_linker %q\n", string(data[:len(data)-1])) + fmt.Fprintf(stdout, "//go:cgo_dynamic_linker %q\n", string(data[:len(data)-1])) } } } @@ -203,14 +241,14 @@ func dynimport(obj string) { if s.Version != "" { targ += "#" + s.Version } - fmt.Fprintf(stdout, "#pragma cgo_import_dynamic %s %s %q\n", s.Name, targ, s.Library) + fmt.Fprintf(stdout, "//go:cgo_import_dynamic %s %s %q\n", s.Name, targ, s.Library) } lib, err := f.ImportedLibraries() if err != nil { fatalf("cannot load imported libraries from ELF file %s: %v", obj, err) } for _, l := range lib { - fmt.Fprintf(stdout, "#pragma cgo_import_dynamic _ _ %q\n", l) + fmt.Fprintf(stdout, "//go:cgo_import_dynamic _ _ %q\n", l) } return } @@ -224,14 +262,14 @@ func dynimport(obj string) { if len(s) > 0 && s[0] == '_' { s = s[1:] } - fmt.Fprintf(stdout, "#pragma cgo_import_dynamic %s %s %q\n", s, s, "") + fmt.Fprintf(stdout, "//go:cgo_import_dynamic %s %s %q\n", s, s, "") } lib, err := f.ImportedLibraries() if err != nil { fatalf("cannot load imported libraries from Mach-O file %s: %v", obj, err) } for _, l := range lib { - fmt.Fprintf(stdout, "#pragma cgo_import_dynamic _ _ %q\n", l) + fmt.Fprintf(stdout, "//go:cgo_import_dynamic _ _ %q\n", l) } return } @@ -244,7 +282,7 @@ func dynimport(obj string) { for _, s := range sym { ss := strings.Split(s, ":") name := strings.Split(ss[0], "@")[0] - fmt.Fprintf(stdout, "#pragma cgo_import_dynamic %s %s %q\n", name, ss[0], strings.ToLower(ss[1])) + fmt.Fprintf(stdout, "//go:cgo_import_dynamic %s %s %q\n", name, ss[0], strings.ToLower(ss[1])) } return } @@ -252,10 +290,10 @@ func dynimport(obj string) { fatalf("cannot parse %s as ELF, Mach-O or PE", obj) } -// Construct a gcc struct matching the 6c argument frame. +// Construct a gcc struct matching the gc argument frame. // Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes. // These assumptions are checked by the gccProlog. -// Also assumes that 6c convention is to word-align the +// Also assumes that gc convention is to word-align the // input and output parameters. func (p *Package) structType(n *Name) (string, int64) { var buf bytes.Buffer @@ -304,7 +342,7 @@ func (p *Package) structType(n *Name) (string, int64) { return buf.String(), off } -func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { +func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name) { name := n.Go gtype := n.FuncType.Go void := gtype.Results == nil || len(gtype.Results.List) == 0 @@ -396,11 +434,11 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { return } - // C wrapper calls into gcc, passing a pointer to the argument frame. - fmt.Fprintf(fc, "#pragma cgo_import_static %s\n", cname) - fmt.Fprintf(fc, "void %s(void*);\n", cname) - fmt.Fprintf(fc, "#pragma dataflag NOPTR\n") - fmt.Fprintf(fc, "void *·%s = %s;\n", cname, cname) + // Wrapper calls into gcc, passing a pointer to the argument frame. + fmt.Fprintf(fgo2, "//go:cgo_import_static %s\n", cname) + fmt.Fprintf(fgo2, "//go:linkname __cgofn_%s %s\n", cname, cname) + fmt.Fprintf(fgo2, "var __cgofn_%s byte\n", cname) + fmt.Fprintf(fgo2, "var %s = unsafe.Pointer(&__cgofn_%s)\n", cname, cname) nret := 0 if !void { @@ -412,7 +450,6 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { } fmt.Fprint(fgo2, "\n") - fmt.Fprintf(fgo2, "var %s unsafe.Pointer\n", cname) conf.Fprint(fgo2, fset, d) fmt.Fprint(fgo2, " {\n") @@ -428,16 +465,20 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { if n.AddError { prefix = "errno := " } - fmt.Fprintf(fgo2, "\t%s_cgo_runtime_cgocall_errno(%s, %s)\n", prefix, cname, arg) + fmt.Fprintf(fgo2, "\t%s_cgo_runtime_cgocall(%s, %s)\n", prefix, cname, arg) if n.AddError { fmt.Fprintf(fgo2, "\tif errno != 0 { r2 = syscall.Errno(errno) }\n") } + fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n") + for i := range d.Type.Params.List { + fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i) + } + fmt.Fprintf(fgo2, "\t}\n") fmt.Fprintf(fgo2, "\treturn\n") fmt.Fprintf(fgo2, "}\n") } -// writeOutput creates stubs for a specific source file to be compiled by 6g -// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.) +// writeOutput creates stubs for a specific source file to be compiled by gc func (p *Package) writeOutput(f *File, srcfile string) { base := srcfile if strings.HasSuffix(base, ".go") { @@ -454,7 +495,7 @@ func (p *Package) writeOutput(f *File, srcfile string) { fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n") conf.Fprint(fgo1, fset, f.AST) - // While we process the vars and funcs, also write 6c and gcc output. + // While we process the vars and funcs, also write gcc output. // Gcc output starts with the preamble. fmt.Fprintf(fgcc, "%s\n", f.Preamble) fmt.Fprintf(fgcc, "%s\n", gccProlog) @@ -516,7 +557,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) { if n.AddError { fmt.Fprintf(fgcc, "\terrno = 0;\n") } - // We're trying to write a gcc struct that matches 6c/8c/5c's layout. + // We're trying to write a gcc struct that matches gc's layout. // Use packed attribute to force no padding in this struct in case // gcc has different packing requirements. fmt.Fprintf(fgcc, "\t%s %v *a = v;\n", ctype, p.packedAttribute()) @@ -612,13 +653,13 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) { } // packedAttribute returns host compiler struct attribute that will be -// used to match 6c/8c/5c's struct layout. For example, on 386 Windows, -// gcc wants to 8-align int64s, but 8c does not. +// used to match gc's struct layout. For example, on 386 Windows, +// gcc wants to 8-align int64s, but gc does not. // Use __gcc_struct__ to work around http://gcc.gnu.org/PR52991 on x86, -// and http://golang.org/issue/5603. +// and https://golang.org/issue/5603. func (p *Package) packedAttribute() string { s := "__attribute__((__packed__" - if !strings.Contains(p.gccBaseCmd()[0], "clang") && (goarch == "amd64" || goarch == "386") { + if !p.GccIsClang && (goarch == "amd64" || goarch == "386") { s += ", __gcc_struct__" } return s + "))" @@ -626,23 +667,19 @@ func (p *Package) packedAttribute() string { // Write out the various stubs we need to support functions exported // from Go so that they are callable from C. -func (p *Package) writeExports(fgo2, fc, fm *os.File) { - fgcc := creat(*objDir + "_cgo_export.c") - fgcch := creat(*objDir + "_cgo_export.h") - - fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n") - fmt.Fprintf(fgcch, "%s\n", p.Preamble) - fmt.Fprintf(fgcch, "%s\n", p.gccExportHeaderProlog()) +func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { + p.writeExportHeader(fgcch) fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n") - fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n") + fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n\n") - fmt.Fprintf(fgcc, "\nextern void crosscall2(void (*fn)(void *, int), void *, int);\n\n") + fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *, int), void *, int);\n") + fmt.Fprintf(fgcc, "extern void _cgo_wait_runtime_init_done();\n\n") for _, exp := range p.ExpFunc { fn := exp.Func - // Construct a gcc struct matching the 6c argument and + // Construct a gcc struct matching the gc argument and // result frame. The gcc struct will be compiled with // __attribute__((packed)) so all padding must be accounted // for explicitly. @@ -728,11 +765,16 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { s += fmt.Sprintf("%s p%d", p.cgoType(atype).C, i) }) s += ")" + + if len(exp.Doc) > 0 { + fmt.Fprintf(fgcch, "\n%s", exp.Doc) + } fmt.Fprintf(fgcch, "\nextern %s;\n", s) fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *, int);\n", cPrefix, exp.ExpName) fmt.Fprintf(fgcc, "\n%s\n", s) fmt.Fprintf(fgcc, "{\n") + fmt.Fprintf(fgcc, "\t_cgo_wait_runtime_init_done();\n") fmt.Fprintf(fgcc, "\t%s %v a;\n", ctype, p.packedAttribute()) if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) @@ -758,20 +800,21 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { } fmt.Fprintf(fgcc, "}\n") - // Build the wrapper function compiled by 6c/8c + // Build the wrapper function compiled by gc. goname := exp.Func.Name.Name if fn.Recv != nil { goname = "_cgoexpwrap" + cPrefix + "_" + fn.Recv.List[0].Names[0].Name + "_" + goname } - fmt.Fprintf(fc, "#pragma cgo_export_dynamic %s\n", goname) - fmt.Fprintf(fc, "extern void ·%s();\n\n", goname) - fmt.Fprintf(fc, "#pragma cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) - fmt.Fprintf(fc, "#pragma textflag 7\n") // no split stack, so no use of m or g - fmt.Fprintf(fc, "void\n") - fmt.Fprintf(fc, "_cgoexp%s_%s(void *a, int32 n)\n", cPrefix, exp.ExpName) - fmt.Fprintf(fc, "{\n") - fmt.Fprintf(fc, "\truntime·cgocallback(·%s, a, n);\n", goname) - fmt.Fprintf(fc, "}\n") + fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", goname) + fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName) + fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g + fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted + fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32) {", cPrefix, exp.ExpName) + fmt.Fprintf(fgo2, "\tfn := %s\n", goname) + // The indirect here is converting from a Go function pointer to a C function pointer. + fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n));\n") + fmt.Fprintf(fgo2, "}\n") fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) @@ -814,23 +857,20 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { fmt.Fprint(fgo2, "}\n") } } + + fmt.Fprintf(fgcch, "%s", gccExportHeaderEpilog) } // Write out the C header allowing C code to call exported gccgo functions. -func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { - fgcc := creat(*objDir + "_cgo_export.c") - fgcch := creat(*objDir + "_cgo_export.h") - +func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { gccgoSymbolPrefix := p.gccgoSymbolPrefix() - fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n") - fmt.Fprintf(fgcch, "%s\n", p.Preamble) - fmt.Fprintf(fgcch, "%s\n", p.gccExportHeaderProlog()) + p.writeExportHeader(fgcch) fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n") fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n") - fmt.Fprintf(fm, "#include \"_cgo_export.h\"\n") + fmt.Fprintf(fgcc, "%s\n", gccgoExportFileProlog) for _, exp := range p.ExpFunc { fn := exp.Func @@ -851,6 +891,7 @@ func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { }) default: // Declare a result struct. + fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName) fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName) forFieldList(fntype.Results, func(i int, atype ast.Expr) { @@ -880,6 +921,10 @@ func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { fmt.Fprintf(cdeclBuf, ")") cParams := cdeclBuf.String() + if len(exp.Doc) > 0 { + fmt.Fprintf(fgcch, "\n%s", exp.Doc) + } + // We need to use a name that will be exported by the // Go code; otherwise gccgo will make it static and we // will not be able to link against it from the C @@ -900,6 +945,8 @@ func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { fmt.Fprint(fgcc, "\n") fmt.Fprintf(fgcc, "%s %s %s {\n", cRet, exp.ExpName, cParams) + fmt.Fprintf(fgcc, "\tif(_cgo_wait_runtime_init_done)\n") + fmt.Fprintf(fgcc, "\t\t_cgo_wait_runtime_init_done();\n") fmt.Fprint(fgcc, "\t") if resultCount > 0 { fmt.Fprint(fgcc, "return ") @@ -919,7 +966,8 @@ func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { fmt.Fprint(fgcc, "}\n") // Dummy declaration for _cgo_main.c - fmt.Fprintf(fm, "%s %s %s {}\n", cRet, goName, cParams) + fmt.Fprintf(fm, `char %s[1] __asm__("%s.%s");`, goName, gccgoSymbolPrefix, goName) + fmt.Fprint(fm, "\n") // For gccgo we use a wrapper function in Go, in order // to call CgocallBack and CgocallBackDone. @@ -974,6 +1022,24 @@ func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { fmt.Fprint(fgo2, ")\n") fmt.Fprint(fgo2, "}\n") } + + fmt.Fprintf(fgcch, "%s", gccExportHeaderEpilog) +} + +// writeExportHeader writes out the start of the _cgo_export.h file. +func (p *Package) writeExportHeader(fgcch io.Writer) { + fmt.Fprintf(fgcch, "/* Created by \"go tool cgo\" - DO NOT EDIT. */\n\n") + pkg := *importPath + if pkg == "" { + pkg = p.PackagePath + } + fmt.Fprintf(fgcch, "/* package %s */\n\n", pkg) + + fmt.Fprintf(fgcch, "/* Start of preamble from import \"C\" comments. */\n\n") + fmt.Fprintf(fgcch, "%s\n", p.Preamble) + fmt.Fprintf(fgcch, "\n/* End of preamble from import \"C\" comments. */\n\n") + + fmt.Fprintf(fgcch, "%s\n", p.gccExportHeaderProlog()) } // Return the package prefix when using gccgo. @@ -1164,60 +1230,39 @@ char *CString(_GoString_); void *_CMalloc(size_t); ` -const cProlog = ` -#include "runtime.h" -#include "cgocall.h" -#include "textflag.h" - -#pragma dataflag NOPTR -static void *cgocall_errno = runtime·cgocall_errno; -#pragma dataflag NOPTR -void *·_cgo_runtime_cgocall_errno = &cgocall_errno; - -#pragma dataflag NOPTR -static void *runtime_gostring = runtime·gostring; -#pragma dataflag NOPTR -void *·_cgo_runtime_gostring = &runtime_gostring; - -#pragma dataflag NOPTR -static void *runtime_gostringn = runtime·gostringn; -#pragma dataflag NOPTR -void *·_cgo_runtime_gostringn = &runtime_gostringn; - -#pragma dataflag NOPTR -static void *runtime_gobytes = runtime·gobytes; -#pragma dataflag NOPTR -void *·_cgo_runtime_gobytes = &runtime_gobytes; - -#pragma dataflag NOPTR -static void *runtime_cmalloc = runtime·cmalloc; -#pragma dataflag NOPTR -void *·_cgo_runtime_cmalloc = &runtime_cmalloc; - -void ·_Cerrno(void*, int32); -` - const goProlog = ` -var _cgo_runtime_cgocall_errno func(unsafe.Pointer, uintptr) int32 -var _cgo_runtime_cmalloc func(uintptr) unsafe.Pointer +//go:linkname _cgo_runtime_cgocall runtime.cgocall +func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32 + +//go:linkname _cgo_runtime_cmalloc runtime.cmalloc +func _cgo_runtime_cmalloc(uintptr) unsafe.Pointer + +//go:linkname _cgo_runtime_cgocallback runtime.cgocallback +func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr) ` const goStringDef = ` -var _cgo_runtime_gostring func(*_Ctype_char) string +//go:linkname _cgo_runtime_gostring runtime.gostring +func _cgo_runtime_gostring(*_Ctype_char) string + func _Cfunc_GoString(p *_Ctype_char) string { return _cgo_runtime_gostring(p) } ` const goStringNDef = ` -var _cgo_runtime_gostringn func(*_Ctype_char, int) string +//go:linkname _cgo_runtime_gostringn runtime.gostringn +func _cgo_runtime_gostringn(*_Ctype_char, int) string + func _Cfunc_GoStringN(p *_Ctype_char, l _Ctype_int) string { return _cgo_runtime_gostringn(p, int(l)) } ` const goBytesDef = ` -var _cgo_runtime_gobytes func(unsafe.Pointer, int) []byte +//go:linkname _cgo_runtime_gobytes runtime.gobytes +func _cgo_runtime_gobytes(unsafe.Pointer, int) []byte + func _Cfunc_GoBytes(p unsafe.Pointer, l _Ctype_int) []byte { return _cgo_runtime_gobytes(p, int(l)) } @@ -1310,6 +1355,11 @@ func (p *Package) gccExportHeaderProlog() string { } const gccExportHeaderProlog = ` +/* Start of boilerplate cgo prologue. */ + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + typedef signed char GoInt8; typedef unsigned char GoUint8; typedef short GoInt16; @@ -1326,9 +1376,44 @@ typedef double GoFloat64; typedef __complex float GoComplex64; typedef __complex double GoComplex128; +// static assertion to make sure the file is being used on architecture +// at least with matching size of GoInt. +typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1]; + typedef struct { char *p; GoInt n; } GoString; typedef void *GoMap; typedef void *GoChan; typedef struct { void *t; void *v; } GoInterface; typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif +` + +// gccExportHeaderEpilog goes at the end of the generated header file. +const gccExportHeaderEpilog = ` +#ifdef __cplusplus +} +#endif +` + +// gccgoExportFileProlog is written to the _cgo_export.c file when +// using gccgo. +// We use weak declarations, and test the addresses, so that this code +// works with older versions of gccgo. +const gccgoExportFileProlog = ` +extern _Bool runtime_iscgo __attribute__ ((weak)); + +static void GoInit(void) __attribute__ ((constructor)); +static void GoInit(void) { + if(&runtime_iscgo) + runtime_iscgo = 1; +} + +extern void _cgo_wait_runtime_init_done() __attribute__ ((weak)); ` diff --git a/libgo/go/cmd/cgo/util.go b/libgo/go/cmd/cgo/util.go index 4e7800d1272..3adb8e87836 100644 --- a/libgo/go/cmd/cgo/util.go +++ b/libgo/go/cmd/cgo/util.go @@ -55,7 +55,7 @@ func error_(pos token.Pos, msg string, args ...interface{}) { fmt.Fprintf(os.Stderr, "\n") } -// isName returns true if s is a valid C identifier +// isName reports whether s is a valid C identifier func isName(s string) bool { for i, v := range s { if v != '_' && (v < 'A' || v > 'Z') && (v < 'a' || v > 'z') && (v < '0' || v > '9') { diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go new file mode 100644 index 00000000000..1134997eaaa --- /dev/null +++ b/libgo/go/cmd/go/alldocs.go @@ -0,0 +1,1481 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DO NOT EDIT THIS FILE. GENERATED BY mkalldocs.sh. +// Edit the documentation in other files and rerun mkalldocs.sh to generate this one. + +/* +Go is a tool for managing Go source code. + +Usage: + + go command [arguments] + +The commands are: + + build compile packages and dependencies + clean remove object files + doc show documentation for package or symbol + env print Go environment information + fix run go tool fix on packages + fmt run gofmt on package sources + generate generate Go files by processing source + get download and install packages and dependencies + install compile and install packages and dependencies + list list packages + run compile and run Go program + test test packages + tool run specified go tool + version print Go version + vet run go tool vet on packages + +Use "go help [command]" for more information about a command. + +Additional help topics: + + c calling between Go and C + buildmode description of build modes + filetype file types + gopath GOPATH environment variable + environment environment variables + importpath import path syntax + packages description of package lists + testflag description of testing flags + testfunc description of testing functions + +Use "go help [topic]" for more information about that topic. + + +Compile packages and dependencies + +Usage: + + go build [-o output] [-i] [build flags] [packages] + +Build compiles the packages named by the import paths, +along with their dependencies, but it does not install the results. + +If the arguments to build are a list of .go files, build treats +them as a list of source files specifying a single package. + +When compiling a single main package, build writes +the resulting executable to an output file named after +the first source file ('go build ed.go rx.go' writes 'ed' or 'ed.exe') +or the source code directory ('go build unix/sam' writes 'sam' or 'sam.exe'). +The '.exe' suffix is added when writing a Windows executable. + +When compiling multiple packages or a single non-main package, +build compiles the packages but discards the resulting object, +serving only as a check that the packages can be built. + +The -o flag, only allowed when compiling a single package, +forces build to write the resulting executable or object +to the named output file, instead of the default behavior described +in the last two paragraphs. + +The -i flag installs the packages that are dependencies of the target. + +The build flags are shared by the build, clean, get, install, list, run, +and test commands: + + -a + force rebuilding of packages that are already up-to-date. + -n + print the commands but do not run them. + -p n + the number of builds that can be run in parallel. + The default is the number of CPUs available, except + on darwin/arm which defaults to 1. + -race + enable data race detection. + Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. + -v + print the names of packages as they are compiled. + -work + print the name of the temporary work directory and + do not delete it when exiting. + -x + print the commands. + + -asmflags 'flag list' + arguments to pass on each go tool asm invocation. + -buildmode mode + build mode to use. See 'go help buildmode' for more. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc). + -gccgoflags 'arg list' + arguments to pass on each gccgo compiler/linker invocation. + -gcflags 'arg list' + arguments to pass on each go tool compile invocation. + -installsuffix suffix + a suffix to use in the name of the package installation directory, + in order to keep output separate from default builds. + If using the -race flag, the install suffix is automatically set to race + or, if set explicitly, has _race appended to it. Using a -buildmode + option that requires non-default compile flags has a similar effect. + -ldflags 'flag list' + arguments to pass on each go tool link invocation. + -linkshared + link against shared libraries previously created with + -buildmode=shared + -pkgdir dir + install and load all packages from dir instead of the usual locations. + For example, when building with a non-standard configuration, + use -pkgdir to keep generated packages in a separate location. + -tags 'tag list' + a list of build tags to consider satisfied during the build. + For more information about build tags, see the description of + build constraints in the documentation for the go/build package. + -toolexec 'cmd args' + a program to use to invoke toolchain programs like vet and asm. + For example, instead of running asm, the go command will run + 'cmd args /path/to/asm <arguments for asm>'. + +The list flags accept a space-separated list of strings. To embed spaces +in an element in the list, surround it with either single or double quotes. + +For more about specifying packages, see 'go help packages'. +For more about where packages and binaries are installed, +run 'go help gopath'. +For more about calling between Go and C/C++, run 'go help c'. + +Note: Build adheres to certain conventions such as those described +by 'go help gopath'. Not all projects can follow these conventions, +however. Installations that have their own conventions or that use +a separate software build system may choose to use lower-level +invocations such as 'go tool compile' and 'go tool link' to avoid +some of the overheads and design decisions of the build tool. + +See also: go install, go get, go clean. + + +Remove object files + +Usage: + + go clean [-i] [-r] [-n] [-x] [build flags] [packages] + +Clean removes object files from package source directories. +The go command builds most objects in a temporary directory, +so go clean is mainly concerned with object files left by other +tools or by manual invocations of go build. + +Specifically, clean removes the following files from each of the +source directories corresponding to the import paths: + + _obj/ old object directory, left from Makefiles + _test/ old test directory, left from Makefiles + _testmain.go old gotest file, left from Makefiles + test.out old test log, left from Makefiles + build.out old test log, left from Makefiles + *.[568ao] object files, left from Makefiles + + DIR(.exe) from go build + DIR.test(.exe) from go test -c + MAINFILE(.exe) from go build MAINFILE.go + *.so from SWIG + +In the list, DIR represents the final path element of the +directory, and MAINFILE is the base name of any Go source +file in the directory that is not included when building +the package. + +The -i flag causes clean to remove the corresponding installed +archive or binary (what 'go install' would create). + +The -n flag causes clean to print the remove commands it would execute, +but not run them. + +The -r flag causes clean to be applied recursively to all the +dependencies of the packages named by the import paths. + +The -x flag causes clean to print remove commands as it executes them. + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + + +Show documentation for package or symbol + +Usage: + + go doc [-u] [-c] [package|[package.]symbol[.method]] + +Doc prints the documentation comments associated with the item identified by its +arguments (a package, const, func, type, var, or method) followed by a one-line +summary of each of the first-level items "under" that item (package-level +declarations for a package, methods for a type, etc.). + +Doc accepts zero, one, or two arguments. + +Given no arguments, that is, when run as + + go doc + +it prints the package documentation for the package in the current directory. +If the package is a command (package main), the exported symbols of the package +are elided from the presentation unless the -cmd flag is provided. + +When run with one argument, the argument is treated as a Go-syntax-like +representation of the item to be documented. What the argument selects depends +on what is installed in GOROOT and GOPATH, as well as the form of the argument, +which is schematically one of these: + + go doc <pkg> + go doc <sym>[.<method>] + go doc [<pkg>].<sym>[.<method>] + +The first item in this list matched by the argument is the one whose +documentation is printed. (See the examples below.) For packages, the order of +scanning is determined lexically, but the GOROOT tree is always scanned before +GOPATH. + +If there is no package specified or matched, the package in the current +directory is selected, so "go doc Foo" shows the documentation for symbol Foo in +the current package. + +The package path must be either a qualified path or a proper suffix of a +path. The go tool's usual package mechanism does not apply: package path +elements like . and ... are not implemented by go doc. + +When run with two arguments, the first must be a full package path (not just a +suffix), and the second is a symbol or symbol and method; this is similar to the +syntax accepted by godoc: + + go doc <pkg> <sym>[.<method>] + +In all forms, when matching symbols, lower-case letters in the argument match +either case but upper-case letters match exactly. This means that there may be +multiple matches of a lower-case argument in a package if different symbols have +different cases. If this occurs, documentation for all matches is printed. + +Examples: + go doc + Show documentation for current package. + go doc Foo + Show documentation for Foo in the current package. + (Foo starts with a capital letter so it cannot match + a package path.) + go doc encoding/json + Show documentation for the encoding/json package. + go doc json + Shorthand for encoding/json. + go doc json.Number (or go doc json.number) + Show documentation and method summary for json.Number. + go doc json.Number.Int64 (or go doc json.number.int64) + Show documentation for json.Number's Int64 method. + go doc cmd/doc + Show package docs for the doc command. + go doc -cmd cmd/doc + Show package docs and exported symbols within the doc command. + go doc template.new + Show documentation for html/template's New function. + (html/template is lexically before text/template) + go doc text/template.new # One argument + Show documentation for text/template's New function. + go doc text/template new # Two arguments + Show documentation for text/template's New function. + +Flags: + -c + Respect case when matching symbols. + -cmd + Treat a command (package main) like a regular package. + Otherwise package main's exported symbols are hidden + when showing the package's top-level documentation. + -u + Show documentation for unexported as well as exported + symbols and methods. + + +Print Go environment information + +Usage: + + go env [var ...] + +Env prints Go environment information. + +By default env prints information as a shell script +(on Windows, a batch file). If one or more variable +names is given as arguments, env prints the value of +each named variable on its own line. + + +Run go tool fix on packages + +Usage: + + go fix [packages] + +Fix runs the Go fix command on the packages named by the import paths. + +For more about fix, see 'go doc cmd/fix'. +For more about specifying packages, see 'go help packages'. + +To run fix with specific options, run 'go tool fix'. + +See also: go fmt, go vet. + + +Run gofmt on package sources + +Usage: + + go fmt [-n] [-x] [packages] + +Fmt runs the command 'gofmt -l -w' on the packages named +by the import paths. It prints the names of the files that are modified. + +For more about gofmt, see 'go doc cmd/gofmt'. +For more about specifying packages, see 'go help packages'. + +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +To run gofmt with specific options, run gofmt itself. + +See also: go fix, go vet. + + +Generate Go files by processing source + +Usage: + + go generate [-run regexp] [file.go... | packages] + +Generate runs commands described by directives within existing +files. Those commands can run any process but the intent is to +create or update Go source files, for instance by running yacc. + +Go generate is never run automatically by go build, go get, go test, +and so on. It must be run explicitly. + +Go generate scans the file for directives, which are lines of +the form, + + //go:generate command argument... + +(note: no leading spaces and no space in "//go") where command +is the generator to be run, corresponding to an executable file +that can be run locally. It must either be in the shell path +(gofmt), a fully qualified path (/usr/you/bin/mytool), or a +command alias, described below. + +Note that go generate does not parse the file, so lines that look +like directives in comments or multiline strings will be treated +as directives. + +The arguments to the directive are space-separated tokens or +double-quoted strings passed to the generator as individual +arguments when it is run. + +Quoted strings use Go syntax and are evaluated before execution; a +quoted string appears as a single argument to the generator. + +Go generate sets several variables when it runs the generator: + + $GOARCH + The execution architecture (arm, amd64, etc.) + $GOOS + The execution operating system (linux, windows, etc.) + $GOFILE + The base name of the file. + $GOLINE + The line number of the directive in the source file. + $GOPACKAGE + The name of the package of the file containing the directive. + $DOLLAR + A dollar sign. + +Other than variable substitution and quoted-string evaluation, no +special processing such as "globbing" is performed on the command +line. + +As a last step before running the command, any invocations of any +environment variables with alphanumeric names, such as $GOFILE or +$HOME, are expanded throughout the command line. The syntax for +variable expansion is $NAME on all operating systems. Due to the +order of evaluation, variables are expanded even inside quoted +strings. If the variable NAME is not set, $NAME expands to the +empty string. + +A directive of the form, + + //go:generate -command xxx args... + +specifies, for the remainder of this source file only, that the +string xxx represents the command identified by the arguments. This +can be used to create aliases or to handle multiword generators. +For example, + + //go:generate -command yacc go tool yacc + +specifies that the command "yacc" represents the generator +"go tool yacc". + +Generate processes packages in the order given on the command line, +one at a time. If the command line lists .go files, they are treated +as a single package. Within a package, generate processes the +source files in a package in file name order, one at a time. Within +a source file, generate runs generators in the order they appear +in the file, one at a time. + +If any generator returns an error exit status, "go generate" skips +all further processing for that package. + +The generator is run in the package's source directory. + +Go generate accepts one specific flag: + + -run="" + if non-empty, specifies a regular expression to select + directives whose full original source text (excluding + any trailing spaces and final newline) matches the + expression. + +It also accepts the standard build flags -v, -n, and -x. +The -v flag prints the names of packages and files as they are +processed. +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about specifying packages, see 'go help packages'. + + +Download and install packages and dependencies + +Usage: + + go get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages] + +Get downloads and installs the packages named by the import paths, +along with their dependencies. + +The -d flag instructs get to stop after downloading the packages; that is, +it instructs get not to install the packages. + +The -f flag, valid only when -u is set, forces get -u not to verify that +each package has been checked out from the source control repository +implied by its import path. This can be useful if the source is a local fork +of the original. + +The -fix flag instructs get to run the fix tool on the downloaded packages +before resolving dependencies or building the code. + +The -insecure flag permits fetching from repositories and resolving +custom domains using insecure schemes such as HTTP. Use with caution. + +The -t flag instructs get to also download the packages required to build +the tests for the specified packages. + +The -u flag instructs get to use the network to update the named packages +and their dependencies. By default, get uses the network to check out +missing packages but does not use it to look for updates to existing packages. + +Get also accepts build flags to control the installation. See 'go help build'. + +When checking out or updating a package, get looks for a branch or tag +that matches the locally installed version of Go. The most important +rule is that if the local installation is running version "go1", get +searches for a branch or tag named "go1". If no such version exists it +retrieves the most recent version of the package. + +If the vendoring experiment is enabled (see 'go help gopath'), +then when go get checks out or updates a Git repository, +it also updates any git submodules referenced by the repository. + +For more about specifying packages, see 'go help packages'. + +For more about how 'go get' finds source code to +download, see 'go help importpath'. + +See also: go build, go install, go clean. + + +Compile and install packages and dependencies + +Usage: + + go install [build flags] [packages] + +Install compiles and installs the packages named by the import paths, +along with their dependencies. + +For more about the build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go get, go clean. + + +List packages + +Usage: + + go list [-e] [-f format] [-json] [build flags] [packages] + +List lists the packages named by the import paths, one per line. + +The default output shows the package import path: + + bytes + encoding/json + github.com/gorilla/mux + golang.org/x/net/html + +The -f flag specifies an alternate format for the list, using the +syntax of package template. The default output is equivalent to -f +'{{.ImportPath}}'. The struct being passed to the template is: + + type Package struct { + Dir string // directory containing package sources + ImportPath string // import path of package in dir + ImportComment string // path in import comment on package statement + Name string // package name + Doc string // package documentation string + Target string // install path + Shlib string // the shared library that contains this package (only set when -linkshared) + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? + Stale bool // would 'go install' do anything for this package? + Root string // Go root or Go path dir containing this package + + // Source files + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go sources files that import "C" + IgnoredGoFiles []string // .go sources ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + + // Cgo directives + CgoCFLAGS []string // cgo: flags for C compiler + CgoCPPFLAGS []string // cgo: flags for C preprocessor + CgoCXXFLAGS []string // cgo: flags for C++ compiler + CgoLDFLAGS []string // cgo: flags for linker + CgoPkgConfig []string // cgo: pkg-config names + + // Dependency information + Imports []string // import paths used by this package + Deps []string // all (recursively) imported dependencies + + // Error information + Incomplete bool // this package or a dependency has an error + Error *PackageError // error loading package + DepsErrors []*PackageError // errors loading dependencies + + TestGoFiles []string // _test.go files in package + TestImports []string // imports from TestGoFiles + XTestGoFiles []string // _test.go files outside package + XTestImports []string // imports from XTestGoFiles + } + +The template function "join" calls strings.Join. + +The template function "context" returns the build context, defined as: + + type Context struct { + GOARCH string // target architecture + GOOS string // target operating system + GOROOT string // Go root + GOPATH string // Go path + CgoEnabled bool // whether cgo can be used + UseAllFiles bool // use files regardless of +build lines, file names + Compiler string // compiler to assume when computing target paths + BuildTags []string // build constraints to match in +build lines + ReleaseTags []string // releases the current release is compatible with + InstallSuffix string // suffix to use in the name of the install dir + } + +For more information about the meaning of these fields see the documentation +for the go/build package's Context type. + +The -json flag causes the package data to be printed in JSON format +instead of using the template format. + +The -e flag changes the handling of erroneous packages, those that +cannot be found or are malformed. By default, the list command +prints an error to standard error for each erroneous package and +omits the packages from consideration during the usual printing. +With the -e flag, the list command never prints errors to standard +error and instead processes the erroneous packages with the usual +printing. Erroneous packages will have a non-empty ImportPath and +a non-nil Error field; other information may or may not be missing +(zeroed). + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + + +Compile and run Go program + +Usage: + + go run [build flags] [-exec xprog] gofiles... [arguments...] + +Run compiles and runs the main package comprising the named Go source files. +A Go source file is defined to be a file ending in a literal ".go" suffix. + +By default, 'go run' runs the compiled binary directly: 'a.out arguments...'. +If the -exec flag is given, 'go run' invokes the binary using xprog: + 'xprog a.out arguments...'. +If the -exec flag is not given, GOOS or GOARCH is different from the system +default, and a program named go_$GOOS_$GOARCH_exec can be found +on the current search path, 'go run' invokes the binary using that program, +for example 'go_nacl_386_exec a.out arguments...'. This allows execution of +cross-compiled programs when a simulator or other execution method is +available. + +For more about build flags, see 'go help build'. + +See also: go build. + + +Test packages + +Usage: + + go test [-c] [-i] [build and test flags] [packages] [flags for test binary] + +'Go test' automates testing the packages named by the import paths. +It prints a summary of the test results in the format: + + ok archive/tar 0.011s + FAIL archive/zip 0.022s + ok compress/gzip 0.033s + ... + +followed by detailed output for each failed package. + +'Go test' recompiles each package along with any files with names matching +the file pattern "*_test.go". +Files whose names begin with "_" (including "_test.go") or "." are ignored. +These additional files can contain test functions, benchmark functions, and +example functions. See 'go help testfunc' for more. +Each listed package causes the execution of a separate test binary. + +Test files that declare a package with the suffix "_test" will be compiled as a +separate package, and then linked and run with the main test binary. + +By default, go test needs no arguments. It compiles and tests the package +with source in the current directory, including tests, and runs the tests. + +The package is built in a temporary directory so it does not interfere with the +non-test installation. + +In addition to the build flags, the flags handled by 'go test' itself are: + + -c + Compile the test binary to pkg.test but do not run it + (where pkg is the last element of the package's import path). + The file name can be changed with the -o flag. + + -exec xprog + Run the test binary using xprog. The behavior is the same as + in 'go run'. See 'go help run' for details. + + -i + Install packages that are dependencies of the test. + Do not run the test. + + -o file + Compile the test binary to the named file. + The test still runs (unless -c or -i is specified). + +The test binary also accepts flags that control execution of the test; these +flags are also accessible by 'go test'. See 'go help testflag' for details. + +If the test binary needs any other flags, they should be presented after the +package names. The go tool treats as a flag the first argument that begins with +a minus sign that it does not recognize itself; that argument and all subsequent +arguments are passed as arguments to the test binary. + +For more about build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go vet. + + +Run specified go tool + +Usage: + + go tool [-n] command [args...] + +Tool runs the go tool command identified by the arguments. +With no arguments it prints the list of known tools. + +The -n flag causes tool to print the command that would be +executed but not execute it. + +For more about each tool command, see 'go tool command -h'. + + +Print Go version + +Usage: + + go version + +Version prints the Go version, as reported by runtime.Version. + + +Run go tool vet on packages + +Usage: + + go vet [-n] [-x] [build flags] [packages] + +Vet runs the Go vet command on the packages named by the import paths. + +For more about vet, see 'go doc cmd/vet'. +For more about specifying packages, see 'go help packages'. + +To run the vet tool with specific options, run 'go tool vet'. + +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about build flags, see 'go help build'. + +See also: go fmt, go fix. + + +Calling between Go and C + +There are two different ways to call between Go and C/C++ code. + +The first is the cgo tool, which is part of the Go distribution. For +information on how to use it see the cgo documentation (go doc cmd/cgo). + +The second is the SWIG program, which is a general tool for +interfacing between languages. For information on SWIG see +http://swig.org/. When running go build, any file with a .swig +extension will be passed to SWIG. Any file with a .swigcxx extension +will be passed to SWIG with the -c++ option. + +When either cgo or SWIG is used, go build will pass any .c, .m, .s, +or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++ +compiler. The CC or CXX environment variables may be set to determine +the C or C++ compiler, respectively, to use. + + +Description of build modes + +The 'go build' and 'go install' commands take a -buildmode argument which +indicates which kind of object file is to be built. Currently supported values +are: + + -buildmode=archive + Build the listed non-main packages into .a files. Packages named + main are ignored. + + -buildmode=c-archive + Build the listed main package, plus all packages it imports, + into a C archive file. The only callable symbols will be those + functions exported using a cgo //export comment. Requires + exactly one main package to be listed. + + -buildmode=c-shared + Build the listed main packages, plus all packages that they + import, into C shared libraries. The only callable symbols will + be those functions exported using a cgo //export comment. + Non-main packages are ignored. + + -buildmode=default + Listed main packages are built into executables and listed + non-main packages are built into .a files (the default + behavior). + + -buildmode=shared + Combine all the listed non-main packages into a single shared + library that will be used when building with the -linkshared + option. Packages named main are ignored. + + -buildmode=exe + Build the listed main packages and everything they import into + executables. Packages not named main are ignored. + + +File types + +The go command examines the contents of a restricted set of files +in each directory. It identifies which files to examine based on +the extension of the file name. These extensions are: + + .go + Go source files. + .c, .h + C source files. + If the package uses cgo or SWIG, these will be compiled with the + OS-native compiler (typically gcc); otherwise they will + trigger an error. + .cc, .cpp, .cxx, .hh, .hpp, .hxx + C++ source files. Only useful with cgo or SWIG, and always + compiled with the OS-native compiler. + .m + Objective-C source files. Only useful with cgo, and always + compiled with the OS-native compiler. + .s, .S + Assembler source files. + If the package uses cgo or SWIG, these will be assembled with the + OS-native assembler (typically gcc (sic)); otherwise they + will be assembled with the Go assembler. + .swig, .swigcxx + SWIG definition files. + .syso + System object files. + +Files of each of these types except .syso may contain build +constraints, but the go command stops scanning for build constraints +at the first item in the file that is not a blank line or //-style +line comment. + + +GOPATH environment variable + +The Go path is used to resolve import statements. +It is implemented by and documented in the go/build package. + +The GOPATH environment variable lists places to look for Go code. +On Unix, the value is a colon-separated string. +On Windows, the value is a semicolon-separated string. +On Plan 9, the value is a list. + +GOPATH must be set to get, build and install packages outside the +standard Go tree. + +Each directory listed in GOPATH must have a prescribed structure: + +The src directory holds source code. The path below src +determines the import path or executable name. + +The pkg directory holds installed package objects. +As in the Go tree, each target operating system and +architecture pair has its own subdirectory of pkg +(pkg/GOOS_GOARCH). + +If DIR is a directory listed in the GOPATH, a package with +source in DIR/src/foo/bar can be imported as "foo/bar" and +has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". + +The bin directory holds compiled commands. +Each command is named for its source directory, but only +the final element, not the entire path. That is, the +command with source in DIR/src/foo/quux is installed into +DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped +so that you can add DIR/bin to your PATH to get at the +installed commands. If the GOBIN environment variable is +set, commands are installed to the directory it names instead +of DIR/bin. + +Here's an example directory layout: + + GOPATH=/home/user/gocode + + /home/user/gocode/ + src/ + foo/ + bar/ (go code in package bar) + x.go + quux/ (go code in package main) + y.go + bin/ + quux (installed command) + pkg/ + linux_amd64/ + foo/ + bar.a (installed package object) + +Go searches each directory listed in GOPATH to find source code, +but new packages are always downloaded into the first directory +in the list. + +See https://golang.org/doc/code.html for an example. + +Internal Directories + +Code in or below a directory named "internal" is importable only +by code in the directory tree rooted at the parent of "internal". +Here's an extended version of the directory layout above: + + /home/user/gocode/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + internal/ + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + + +The code in z.go is imported as "foo/internal/baz", but that +import statement can only appear in source files in the subtree +rooted at foo. The source files foo/f.go, foo/bar/x.go, and +foo/quux/y.go can all import "foo/internal/baz", but the source file +crash/bang/b.go cannot. + +See https://golang.org/s/go14internal for details. + +Vendor Directories + +Go 1.5 includes experimental support for using local copies +of external dependencies to satisfy imports of those dependencies, +often referred to as vendoring. Setting the environment variable +GO15VENDOREXPERIMENT=1 enables that experimental support. + +When the vendor experiment is enabled, +code below a directory named "vendor" is importable only +by code in the directory tree rooted at the parent of "vendor", +and only using an import path that omits the prefix up to and +including the vendor element. + +Here's the example from the previous section, +but with the "internal" directory renamed to "vendor" +and a new foo/vendor/crash/bang directory added: + + /home/user/gocode/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + vendor/ + crash/ + bang/ (go code in package bang) + b.go + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + +The same visibility rules apply as for internal, but the code +in z.go is imported as "baz", not as "foo/vendor/baz". + +Code in vendor directories deeper in the source tree shadows +code in higher directories. Within the subtree rooted at foo, an import +of "crash/bang" resolves to "foo/vendor/crash/bang", not the +top-level "crash/bang". + +Code in vendor directories is not subject to import path +checking (see 'go help importpath'). + +When the vendor experiment is enabled, 'go get' checks out +submodules when checking out or updating a git repository +(see 'go help get'). + +The vendoring semantics are an experiment, and they may change +in future releases. Once settled, they will be on by default. + +See https://golang.org/s/go15vendor for details. + + +Environment variables + +The go command, and the tools it invokes, examine a few different +environment variables. For many of these, you can see the default +value of on your system by running 'go env NAME', where NAME is the +name of the variable. + +General-purpose environment variables: + + GCCGO + The gccgo command to run for 'go build -compiler=gccgo'. + GOARCH + The architecture, or processor, for which to compile code. + Examples are amd64, 386, arm, ppc64. + GOBIN + The directory where 'go install' will install a command. + GOOS + The operating system for which to compile code. + Examples are linux, darwin, windows, netbsd. + GOPATH + See 'go help gopath'. + GORACE + Options for the race detector. + See https://golang.org/doc/articles/race_detector.html. + GOROOT + The root of the go tree. + +Environment variables for use with cgo: + + CC + The command to use to compile C code. + CGO_ENABLED + Whether the cgo command is supported. Either 0 or 1. + CGO_CFLAGS + Flags that cgo will pass to the compiler when compiling + C code. + CGO_CPPFLAGS + Flags that cgo will pass to the compiler when compiling + C or C++ code. + CGO_CXXFLAGS + Flags that cgo will pass to the compiler when compiling + C++ code. + CGO_LDFLAGS + Flags that cgo will pass to the compiler when linking. + CXX + The command to use to compile C++ code. + +Architecture-specific environment variables: + + GOARM + For GOARCH=arm, the ARM architecture for which to compile. + Valid values are 5, 6, 7. + GO386 + For GOARCH=386, the floating point instruction set. + Valid values are 387, sse2. + +Special-purpose environment variables: + + GOROOT_FINAL + The root of the installed Go tree, when it is + installed in a location other than where it is built. + File names in stack traces are rewritten from GOROOT to + GOROOT_FINAL. + GO15VENDOREXPERIMENT + Set to 1 to enable the Go 1.5 vendoring experiment. + GO_EXTLINK_ENABLED + Whether the linker should use external linking mode + when using -linkmode=auto with code that uses cgo. + Set to 0 to disable external linking mode, 1 to enable it. + + +Import path syntax + +An import path (see 'go help packages') denotes a package +stored in the local file system. In general, an import path denotes +either a standard package (such as "unicode/utf8") or a package +found in one of the work spaces (see 'go help gopath'). + +Relative import paths + +An import path beginning with ./ or ../ is called a relative path. +The toolchain supports relative import paths as a shortcut in two ways. + +First, a relative path can be used as a shorthand on the command line. +If you are working in the directory containing the code imported as +"unicode" and want to run the tests for "unicode/utf8", you can type +"go test ./utf8" instead of needing to specify the full path. +Similarly, in the reverse situation, "go test .." will test "unicode" from +the "unicode/utf8" directory. Relative patterns are also allowed, like +"go test ./..." to test all subdirectories. See 'go help packages' for details +on the pattern syntax. + +Second, if you are compiling a Go program not in a work space, +you can use a relative path in an import statement in that program +to refer to nearby code also not in a work space. +This makes it easy to experiment with small multipackage programs +outside of the usual work spaces, but such programs cannot be +installed with "go install" (there is no work space in which to install them), +so they are rebuilt from scratch each time they are built. +To avoid ambiguity, Go programs cannot use relative import paths +within a work space. + +Remote import paths + +Certain import paths also +describe how to obtain the source code for the package using +a revision control system. + +A few common code hosting sites have special syntax: + + Bitbucket (Git, Mercurial) + + import "bitbucket.org/user/project" + import "bitbucket.org/user/project/sub/directory" + + GitHub (Git) + + import "github.com/user/project" + import "github.com/user/project/sub/directory" + + Google Code Project Hosting (Git, Mercurial, Subversion) + + import "code.google.com/p/project" + import "code.google.com/p/project/sub/directory" + + import "code.google.com/p/project.subrepository" + import "code.google.com/p/project.subrepository/sub/directory" + + Launchpad (Bazaar) + + import "launchpad.net/project" + import "launchpad.net/project/series" + import "launchpad.net/project/series/sub/directory" + + import "launchpad.net/~user/project/branch" + import "launchpad.net/~user/project/branch/sub/directory" + + IBM DevOps Services (Git) + + import "hub.jazz.net/git/user/project" + import "hub.jazz.net/git/user/project/sub/directory" + +For code hosted on other servers, import paths may either be qualified +with the version control type, or the go tool can dynamically fetch +the import path over https/http and discover where the code resides +from a <meta> tag in the HTML. + +To declare the code location, an import path of the form + + repository.vcs/path + +specifies the given repository, with or without the .vcs suffix, +using the named version control system, and then the path inside +that repository. The supported version control systems are: + + Bazaar .bzr + Git .git + Mercurial .hg + Subversion .svn + +For example, + + import "example.org/user/foo.hg" + +denotes the root directory of the Mercurial repository at +example.org/user/foo or foo.hg, and + + import "example.org/repo.git/foo/bar" + +denotes the foo/bar directory of the Git repository at +example.org/repo or repo.git. + +When a version control system supports multiple protocols, +each is tried in turn when downloading. For example, a Git +download tries https://, then git+ssh://. + +If the import path is not a known code hosting site and also lacks a +version control qualifier, the go tool attempts to fetch the import +over https/http and looks for a <meta> tag in the document's HTML +<head>. + +The meta tag has the form: + + <meta name="go-import" content="import-prefix vcs repo-root"> + +The import-prefix is the import path corresponding to the repository +root. It must be a prefix or an exact match of the package being +fetched with "go get". If it's not an exact match, another http +request is made at the prefix to verify the <meta> tags match. + +The meta tag should appear as early in the file as possible. +In particular, it should appear before any raw JavaScript or CSS, +to avoid confusing the go command's restricted parser. + +The vcs is one of "git", "hg", "svn", etc, + +The repo-root is the root of the version control system +containing a scheme and not containing a .vcs qualifier. + +For example, + + import "example.org/pkg/foo" + +will result in the following requests: + + https://example.org/pkg/foo?go-get=1 (preferred) + http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure) + +If that page contains the meta tag + + <meta name="go-import" content="example.org git https://code.org/r/p/exproj"> + +the go tool will verify that https://example.org/?go-get=1 contains the +same meta tag and then git clone https://code.org/r/p/exproj into +GOPATH/src/example.org. + +New downloaded packages are written to the first directory +listed in the GOPATH environment variable (see 'go help gopath'). + +The go command attempts to download the version of the +package appropriate for the Go release being used. +Run 'go help get' for more. + +Import path checking + +When the custom import path feature described above redirects to a +known code hosting site, each of the resulting packages has two possible +import paths, using the custom domain or the known hosting site. + +A package statement is said to have an "import comment" if it is immediately +followed (before the next newline) by a comment of one of these two forms: + + package math // import "path" + package math /* import "path" * / + +The go command will refuse to install a package with an import comment +unless it is being referred to by that import path. In this way, import comments +let package authors make sure the custom import path is used and not a +direct path to the underlying code hosting site. + +If the vendoring experiment is enabled (see 'go help gopath'), +then import path checking is disabled for code found within vendor trees. +This makes it possible to copy code into alternate locations in vendor trees +without needing to update import comments. + +See https://golang.org/s/go14customimport for details. + + +Description of package lists + +Many commands apply to a set of packages: + + go action [packages] + +Usually, [packages] is a list of import paths. + +An import path that is a rooted path or that begins with +a . or .. element is interpreted as a file system path and +denotes the package in that directory. + +Otherwise, the import path P denotes the package found in +the directory DIR/src/P for some DIR listed in the GOPATH +environment variable (see 'go help gopath'). + +If no import paths are given, the action applies to the +package in the current directory. + +There are four reserved names for paths that should not be used +for packages to be built with the go tool: + +- "main" denotes the top-level package in a stand-alone executable. + +- "all" expands to all package directories found in all the GOPATH +trees. For example, 'go list all' lists all the packages on the local +system. + +- "std" is like all but expands to just the packages in the standard +Go library. + +- "cmd" expands to the Go repository's commands and their +internal libraries. + +An import path is a pattern if it includes one or more "..." wildcards, +each of which can match any string, including the empty string and +strings containing slashes. Such a pattern expands to all package +directories found in the GOPATH trees with names matching the +patterns. As a special case, x/... matches x as well as x's subdirectories. +For example, net/... expands to net and packages in its subdirectories. + +An import path can also name a package to be downloaded from +a remote repository. Run 'go help importpath' for details. + +Every package in a program must have a unique import path. +By convention, this is arranged by starting each path with a +unique prefix that belongs to you. For example, paths used +internally at Google all begin with 'google', and paths +denoting remote repositories begin with the path to the code, +such as 'github.com/user/repo'. + +As a special case, if the package list is a list of .go files from a +single directory, the command is applied to a single synthesized +package made up of exactly those files, ignoring any build constraints +in those files and ignoring any other files in the directory. + +Directory and file names that begin with "." or "_" are ignored +by the go tool, as are directories named "testdata". + + +Description of testing flags + +The 'go test' command takes both flags that apply to 'go test' itself +and flags that apply to the resulting test binary. + +Several of the flags control profiling and write an execution profile +suitable for "go tool pprof"; run "go tool pprof -h" for more +information. The --alloc_space, --alloc_objects, and --show_bytes +options of pprof control how the information is presented. + +The following flags are recognized by the 'go test' command and +control the execution of any test: + + -bench regexp + Run benchmarks matching the regular expression. + By default, no benchmarks run. To run all benchmarks, + use '-bench .' or '-bench=.'. + + -benchmem + Print memory allocation statistics for benchmarks. + + -benchtime t + Run enough iterations of each benchmark to take t, specified + as a time.Duration (for example, -benchtime 1h30s). + The default is 1 second (1s). + + -blockprofile block.out + Write a goroutine blocking profile to the specified file + when all tests are complete. + Writes test binary as -c would. + + -blockprofilerate n + Control the detail provided in goroutine blocking profiles by + calling runtime.SetBlockProfileRate with n. + See 'go doc runtime.SetBlockProfileRate'. + The profiler aims to sample, on average, one blocking event every + n nanoseconds the program spends blocked. By default, + if -test.blockprofile is set without this flag, all blocking events + are recorded, equivalent to -test.blockprofilerate=1. + + -count n + Run each test and benchmark n times (default 1). + If -cpu is set, run n times for each GOMAXPROCS value. + Examples are always run once. + + -cover + Enable coverage analysis. + + -covermode set,count,atomic + Set the mode for coverage analysis for the package[s] + being tested. The default is "set" unless -race is enabled, + in which case it is "atomic". + The values: + set: bool: does this statement run? + count: int: how many times does this statement run? + atomic: int: count, but correct in multithreaded tests; + significantly more expensive. + Sets -cover. + + -coverpkg pkg1,pkg2,pkg3 + Apply coverage analysis in each test to the given list of packages. + The default is for each test to analyze only the package being tested. + Packages are specified as import paths. + Sets -cover. + + -coverprofile cover.out + Write a coverage profile to the file after all tests have passed. + Sets -cover. + + -cpu 1,2,4 + Specify a list of GOMAXPROCS values for which the tests or + benchmarks should be executed. The default is the current value + of GOMAXPROCS. + + -cpuprofile cpu.out + Write a CPU profile to the specified file before exiting. + Writes test binary as -c would. + + -memprofile mem.out + Write a memory profile to the file after all tests have passed. + Writes test binary as -c would. + + -memprofilerate n + Enable more precise (and expensive) memory profiles by setting + runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. + To profile all memory allocations, use -test.memprofilerate=1 + and pass --alloc_space flag to the pprof tool. + + -outputdir directory + Place output files from profiling in the specified directory, + by default the directory in which "go test" is running. + + -parallel n + Allow parallel execution of test functions that call t.Parallel. + The value of this flag is the maximum number of tests to run + simultaneously; by default, it is set to the value of GOMAXPROCS. + + -run regexp + Run only those tests and examples matching the regular + expression. + + -short + Tell long-running tests to shorten their run time. + It is off by default but set during all.bash so that installing + the Go tree can run a sanity check but not spend time running + exhaustive tests. + + -timeout t + If a test runs longer than t, panic. + The default is 10 minutes (10m). + + -trace trace.out + Write an execution trace to the specified file before exiting. + Writes test binary as -c would. + + -v + Verbose output: log all tests as they are run. Also print all + text from Log and Logf calls even if the test succeeds. + +The test binary, called pkg.test where pkg is the name of the +directory containing the package sources, can be invoked directly +after building it with 'go test -c'. When invoking the test binary +directly, each of the standard flag names must be prefixed with 'test.', +as in -test.run=TestMyFunc or -test.v. + +When running 'go test', flags not listed above are passed through +unaltered. For instance, the command + + go test -x -v -cpuprofile=prof.out -dir=testdata -update + +will compile the test binary and then run it as + + pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update + +The test flags that generate profiles (other than for coverage) also +leave the test binary in pkg.test for use when analyzing the profiles. + +Flags not recognized by 'go test' must be placed after any specified packages. + + +Description of testing functions + +The 'go test' command expects to find test, benchmark, and example functions +in the "*_test.go" files corresponding to the package under test. + +A test function is one named TestXXX (where XXX is any alphanumeric string +not starting with a lower case letter) and should have the signature, + + func TestXXX(t *testing.T) { ... } + +A benchmark function is one named BenchmarkXXX and should have the signature, + + func BenchmarkXXX(b *testing.B) { ... } + +An example function is similar to a test function but, instead of using +*testing.T to report success or failure, prints output to os.Stdout. +That output is compared against the function's "Output:" comment, which +must be the last comment in the function body (see example below). An +example with no such comment, or with no text after "Output:" is compiled +but not executed. + +Godoc displays the body of ExampleXXX to demonstrate the use +of the function, constant, or variable XXX. An example of a method M with +receiver type T or *T is named ExampleT_M. There may be multiple examples +for a given function, constant, or variable, distinguished by a trailing _xxx, +where xxx is a suffix not beginning with an upper case letter. + +Here is an example of an example: + + func ExamplePrintln() { + Println("The output of\nthis example.") + // Output: The output of + // this example. + } + +The entire test file is presented as the example when it contains a single +example function, at least one other function, type, variable, or constant +declaration, and no test or benchmark functions. + +See the documentation of the testing package for more information. + + +*/ +package main diff --git a/libgo/go/cmd/go/bootstrap.go b/libgo/go/cmd/go/bootstrap.go index dc7ed5f4c06..1686df77afd 100644 --- a/libgo/go/cmd/go/bootstrap.go +++ b/libgo/go/cmd/go/bootstrap.go @@ -17,11 +17,19 @@ import ( var errHTTP = errors.New("no http in bootstrap go command") +type httpError struct { + statusCode int +} + +func (e *httpError) Error() string { + panic("unreachable") +} + func httpGET(url string) ([]byte, error) { return nil, errHTTP } -func httpsOrHTTP(importPath string) (string, io.ReadCloser, error) { +func httpsOrHTTP(importPath string, security securityMode) (string, io.ReadCloser, error) { return "", nil, errHTTP } diff --git a/libgo/go/cmd/go/build.go b/libgo/go/cmd/go/build.go index 781a43b5d99..3afac2ee062 100644 --- a/libgo/go/cmd/go/build.go +++ b/libgo/go/cmd/go/build.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "container/heap" + "debug/elf" "errors" "flag" "fmt" @@ -34,21 +35,23 @@ var cmdBuild = &Command{ Build compiles the packages named by the import paths, along with their dependencies, but it does not install the results. -If the arguments are a list of .go files, build treats them as a list -of source files specifying a single package. +If the arguments to build are a list of .go files, build treats +them as a list of source files specifying a single package. -When the command line specifies a single main package, -build writes the resulting executable to output. -Otherwise build compiles the packages but discards the results, +When compiling a single main package, build writes +the resulting executable to an output file named after +the first source file ('go build ed.go rx.go' writes 'ed' or 'ed.exe') +or the source code directory ('go build unix/sam' writes 'sam' or 'sam.exe'). +The '.exe' suffix is added when writing a Windows executable. + +When compiling multiple packages or a single non-main package, +build compiles the packages but discards the resulting object, serving only as a check that the packages can be built. -The -o flag specifies the output file name. If not specified, the -output file name depends on the arguments and derives from the name -of the package, such as p.a for package p, unless p is 'main'. If -the package is main and file names are provided, the file name -derives from the first file name mentioned, such as f1 for 'go build -f1.go f2.go'; with no files provided ('go build'), the output file -name is the base name of the containing directory. +The -o flag, only allowed when compiling a single package, +forces build to write the resulting executable or object +to the named output file, instead of the default behavior described +in the last two paragraphs. The -i flag installs the packages that are dependencies of the target. @@ -57,12 +60,12 @@ and test commands: -a force rebuilding of packages that are already up-to-date. - In Go releases, does not apply to the standard library. -n print the commands but do not run them. -p n the number of builds that can be run in parallel. - The default is the number of CPUs available. + The default is the number of CPUs available, except + on darwin/arm which defaults to 1. -race enable data race detection. Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. @@ -74,33 +77,54 @@ and test commands: -x print the commands. - -ccflags 'arg list' - arguments to pass on each 5c, 6c, or 8c compiler invocation. + -asmflags 'flag list' + arguments to pass on each go tool asm invocation. + -buildmode mode + build mode to use. See 'go help buildmode' for more. -compiler name name of compiler to use, as in runtime.Compiler (gccgo or gc). -gccgoflags 'arg list' arguments to pass on each gccgo compiler/linker invocation. -gcflags 'arg list' - arguments to pass on each 5g, 6g, or 8g compiler invocation. + arguments to pass on each go tool compile invocation. -installsuffix suffix a suffix to use in the name of the package installation directory, in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to race - or, if set explicitly, has _race appended to it. + or, if set explicitly, has _race appended to it. Using a -buildmode + option that requires non-default compile flags has a similar effect. -ldflags 'flag list' - arguments to pass on each 5l, 6l, or 8l linker invocation. + arguments to pass on each go tool link invocation. + -linkshared + link against shared libraries previously created with + -buildmode=shared + -pkgdir dir + install and load all packages from dir instead of the usual locations. + For example, when building with a non-standard configuration, + use -pkgdir to keep generated packages in a separate location. -tags 'tag list' a list of build tags to consider satisfied during the build. For more information about build tags, see the description of build constraints in the documentation for the go/build package. + -toolexec 'cmd args' + a program to use to invoke toolchain programs like vet and asm. + For example, instead of running asm, the go command will run + 'cmd args /path/to/asm <arguments for asm>'. The list flags accept a space-separated list of strings. To embed spaces in an element in the list, surround it with either single or double quotes. For more about specifying packages, see 'go help packages'. For more about where packages and binaries are installed, -run 'go help gopath'. For more about calling between Go and C/C++, -run 'go help c'. +run 'go help gopath'. +For more about calling between Go and C/C++, run 'go help c'. + +Note: Build adheres to certain conventions such as those described +by 'go help gopath'. Not all projects can follow these conventions, +however. Installations that have their own conventions or that use +a separate software build system may choose to use lower-level +invocations such as 'go tool compile' and 'go tool link' to avoid +some of the overheads and design decisions of the build tool. See also: go install, go get, go clean. `, @@ -115,6 +139,17 @@ func init() { addBuildFlags(cmdBuild) addBuildFlags(cmdInstall) + + if buildContext.GOOS == "darwin" { + switch buildContext.GOARCH { + case "arm", "arm64": + // darwin/arm cannot run multiple tests simultaneously. + // Parallelism is limited in go_darwin_arm_exec, but + // also needs to be limited here so go test std does not + // timeout tests that waiting to run. + buildP = 1 + } + } } // Flags set by multiple commands. @@ -126,16 +161,21 @@ var buildX bool // -x flag var buildI bool // -i flag var buildO = cmdBuild.Flag.String("o", "", "output file") var buildWork bool // -work flag +var buildAsmflags []string // -asmflags flag var buildGcflags []string // -gcflags flag -var buildCcflags []string // -ccflags flag var buildLdflags []string // -ldflags flag var buildGccgoflags []string // -gccgoflags flag var buildRace bool // -race flag +var buildToolExec []string // -toolexec flag +var buildBuildmode string // -buildmode flag +var buildLinkshared bool // -linkshared flag +var buildPkgdir string // -pkgdir flag // Require the source for go std packages var reqStdPkgSrc bool var buildContext = build.Default var buildToolchain toolchain = noToolchain{} +var ldBuildmode string // buildCompiler implements flag.Var. // It implements Set by updating both @@ -171,21 +211,25 @@ func init() { // addBuildFlags adds the flags common to the build, clean, get, // install, list, run, and test commands. func addBuildFlags(cmd *Command) { - // NOTE: If you add flags here, also add them to testflag.go. cmd.Flag.BoolVar(&buildA, "a", false, "") cmd.Flag.BoolVar(&buildN, "n", false, "") cmd.Flag.IntVar(&buildP, "p", buildP, "") - cmd.Flag.StringVar(&buildContext.InstallSuffix, "installsuffix", "", "") cmd.Flag.BoolVar(&buildV, "v", false, "") cmd.Flag.BoolVar(&buildX, "x", false, "") - cmd.Flag.BoolVar(&buildWork, "work", false, "") + + cmd.Flag.Var((*stringsFlag)(&buildAsmflags), "asmflags", "") + cmd.Flag.Var(buildCompiler{}, "compiler", "") + cmd.Flag.StringVar(&buildBuildmode, "buildmode", "default", "") cmd.Flag.Var((*stringsFlag)(&buildGcflags), "gcflags", "") - cmd.Flag.Var((*stringsFlag)(&buildCcflags), "ccflags", "") - cmd.Flag.Var((*stringsFlag)(&buildLdflags), "ldflags", "") cmd.Flag.Var((*stringsFlag)(&buildGccgoflags), "gccgoflags", "") - cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "") - cmd.Flag.Var(buildCompiler{}, "compiler", "") + cmd.Flag.StringVar(&buildContext.InstallSuffix, "installsuffix", "", "") + cmd.Flag.Var((*stringsFlag)(&buildLdflags), "ldflags", "") + cmd.Flag.BoolVar(&buildLinkshared, "linkshared", false, "") + cmd.Flag.StringVar(&buildPkgdir, "pkgdir", "", "") cmd.Flag.BoolVar(&buildRace, "race", false, "") + cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "") + cmd.Flag.Var((*stringsFlag)(&buildToolExec), "toolexec", "") + cmd.Flag.BoolVar(&buildWork, "work", false, "") switch build.Default.Compiler { case "gc": reqStdPkgSrc = true @@ -266,8 +310,113 @@ func (v *stringsFlag) String() string { return "<stringsFlag>" } +func pkgsMain(pkgs []*Package) (res []*Package) { + for _, p := range pkgs { + if p.Name == "main" { + res = append(res, p) + } + } + return res +} + +func pkgsNotMain(pkgs []*Package) (res []*Package) { + for _, p := range pkgs { + if p.Name != "main" { + res = append(res, p) + } + } + return res +} + +var pkgsFilter = func(pkgs []*Package) []*Package { return pkgs } + +func buildModeInit() { + _, gccgo := buildToolchain.(gccgoToolchain) + var codegenArg string + platform := goos + "/" + goarch + switch buildBuildmode { + case "archive": + pkgsFilter = pkgsNotMain + case "c-archive": + pkgsFilter = func(p []*Package) []*Package { + if len(p) != 1 || p[0].Name != "main" { + fatalf("-buildmode=c-archive requires exactly one main package") + } + return p + } + exeSuffix = ".a" + ldBuildmode = "c-archive" + case "c-shared": + pkgsFilter = pkgsMain + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/amd64": + codegenArg = "-shared" + case "linux/arm": + buildAsmflags = append(buildAsmflags, "-shared") + case "darwin/amd64": + case "android/arm": + default: + fatalf("-buildmode=c-shared not supported on %s\n", platform) + } + } + ldBuildmode = "c-shared" + case "default": + ldBuildmode = "exe" + case "exe": + pkgsFilter = pkgsMain + ldBuildmode = "exe" + case "shared": + pkgsFilter = pkgsNotMain + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/amd64": + default: + fatalf("-buildmode=shared not supported on %s\n", platform) + } + codegenArg = "-dynlink" + } + if *buildO != "" { + fatalf("-buildmode=shared and -o not supported together") + } + ldBuildmode = "shared" + default: + fatalf("buildmode=%s not supported", buildBuildmode) + } + if buildLinkshared { + if gccgo { + codegenArg = "-fPIC" + } else { + if platform != "linux/amd64" { + fmt.Fprintf(os.Stderr, "go %s: -linkshared is only supported on linux/amd64\n", flag.Args()[0]) + os.Exit(2) + } + codegenArg = "-dynlink" + // TODO(mwhudson): remove -w when that gets fixed in linker. + buildLdflags = append(buildLdflags, "-linkshared", "-w") + } + } + if codegenArg != "" { + if gccgo { + buildGccgoflags = append(buildGccgoflags, codegenArg) + } else { + buildAsmflags = append(buildAsmflags, codegenArg) + buildGcflags = append(buildGcflags, codegenArg) + } + if buildContext.InstallSuffix != "" { + buildContext.InstallSuffix += "_" + } + buildContext.InstallSuffix += codegenArg[1:] + } +} + func runBuild(cmd *Command, args []string) { raceInit() + buildModeInit() var b builder b.init() @@ -305,16 +454,21 @@ func runBuild(cmd *Command, args []string) { fatalf("no packages to build") } p := pkgs[0] - p.target = "" // must build - not up to date + p.target = *buildO + p.Stale = true // must build - not up to date a := b.action(modeInstall, depMode, p) - a.target = *buildO b.do(a) return } - a := &action{} - for _, p := range packages(args) { - a.deps = append(a.deps, b.action(modeBuild, depMode, p)) + var a *action + if buildBuildmode == "shared" { + a = b.libaction(libname(args), pkgsFilter(packages(args)), modeBuild, depMode) + } else { + a = &action{} + for _, p := range pkgsFilter(packages(args)) { + a.deps = append(a.deps, b.action(modeBuild, depMode, p)) + } } b.do(a) } @@ -333,18 +487,47 @@ See also: go build, go get, go clean. `, } +// libname returns the filename to use for the shared library when using +// -buildmode=shared. The rules we use are: +// 1) Drop any trailing "/..."s if present +// 2) Change / to - +// 3) Join arguments with , +// So std -> libstd.so +// a b/... -> liba,b.so +// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so +func libname(args []string) string { + var libname string + for _, arg := range args { + arg = strings.TrimSuffix(arg, "/...") + arg = strings.Replace(arg, "/", "-", -1) + if libname == "" { + libname = arg + } else { + libname += "," + arg + } + } + // TODO(mwhudson): Needs to change for platforms that use different naming + // conventions... + return "lib" + libname + ".so" +} + func runInstall(cmd *Command, args []string) { raceInit() - pkgs := packagesForBuild(args) + buildModeInit() + pkgs := pkgsFilter(packagesForBuild(args)) for _, p := range pkgs { if p.Target == "" && (!p.Standard || p.ImportPath != "unsafe") { - if p.cmdline { + switch { + case p.gobinSubdir: + errorf("go install: cannot install cross-compiled binaries when GOBIN is set") + case p.cmdline: errorf("go install: no install location for .go files listed on command line (GOBIN not set)") - } else if p.ConflictDir != "" { + case p.ConflictDir != "": errorf("go install: no install location for %s: hidden by %s", p.Dir, p.ConflictDir) - } else { - errorf("go install: no install location for directory %s outside GOPATH", p.Dir) + default: + errorf("go install: no install location for directory %s outside GOPATH\n"+ + "\tFor more details see: go help gopath", p.Dir) } } } @@ -352,18 +535,68 @@ func runInstall(cmd *Command, args []string) { var b builder b.init() - a := &action{} - for _, p := range pkgs { - a.deps = append(a.deps, b.action(modeInstall, modeInstall, p)) + var a *action + if buildBuildmode == "shared" { + a = b.libaction(libname(args), pkgs, modeInstall, modeInstall) + } else { + a = &action{} + var tools []*action + for _, p := range pkgs { + // If p is a tool, delay the installation until the end of the build. + // This avoids installing assemblers/compilers that are being executed + // by other steps in the build. + // cmd/cgo is handled specially in b.action, so that we can + // both build and use it in the same 'go install'. + action := b.action(modeInstall, modeInstall, p) + if goTools[p.ImportPath] == toTool && p.ImportPath != "cmd/cgo" { + a.deps = append(a.deps, action.deps...) + action.deps = append(action.deps, a) + tools = append(tools, action) + continue + } + a.deps = append(a.deps, action) + } + if len(tools) > 0 { + a = &action{ + deps: tools, + } + } } b.do(a) + exitIfErrors() + + // Success. If this command is 'go install' with no arguments + // and the current directory (the implicit argument) is a command, + // remove any leftover command binary from a previous 'go build'. + // The binary is installed; it's not needed here anymore. + // And worse it might be a stale copy, which you don't want to find + // instead of the installed one if $PATH contains dot. + // One way to view this behavior is that it is as if 'go install' first + // runs 'go build' and the moves the generated file to the install dir. + // See issue 9645. + if len(args) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { + // Compute file 'go build' would have created. + // If it exists and is an executable file, remove it. + _, targ := filepath.Split(pkgs[0].ImportPath) + targ += exeSuffix + if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory + fi, err := os.Stat(targ) + if err == nil { + m := fi.Mode() + if m.IsRegular() { + if m&0111 != 0 || goos == "windows" { // windows never sets executable bit + os.Remove(targ) + } + } + } + } + } } // Global build parameters (used during package load) var ( goarch string goos string - archChar string exeSuffix string ) @@ -373,16 +606,6 @@ func init() { if goos == "windows" { exeSuffix = ".exe" } - var err error - archChar, err = build.ArchChar(goarch) - if err != nil { - if _, isgc := buildToolchain.(gcToolchain); isgc { - fatalf("%s", err) - } - // archChar is only required for gcToolchain, if we're using - // another toolchain leave it blank. - archChar = "" - } } // A builder holds global state about a build. @@ -429,8 +652,9 @@ type action struct { // cacheKey is the key for the action cache. type cacheKey struct { - mode buildMode - p *Package + mode buildMode + p *Package + shlib string } // buildMode specifies the build mode: @@ -505,6 +729,9 @@ func goFilesPackage(gofiles []string) *Package { fatalf("%s is a directory, should be a Go file", file) } dir1, _ := filepath.Split(file) + if dir1 == "" { + dir1 = "./" + } if dir == "" { dir = dir1 } else if dir != dir1 { @@ -541,11 +768,8 @@ func goFilesPackage(gofiles []string) *Package { if gobin != "" { pkg.target = filepath.Join(gobin, exe) } - } else { - if *buildO == "" { - *buildO = pkg.Name + ".a" - } } + pkg.Target = pkg.target pkg.Stale = true @@ -553,24 +777,88 @@ func goFilesPackage(gofiles []string) *Package { return pkg } +// readpkglist returns the list of packages that were built into the shared library +// at shlibpath. For the native toolchain this list is stored, newline separated, in +// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the +// .go_export section. +func readpkglist(shlibpath string) (pkgs []*Package) { + var stk importStack + if _, gccgo := buildToolchain.(gccgoToolchain); gccgo { + f, _ := elf.Open(shlibpath) + sect := f.Section(".go_export") + data, _ := sect.Data() + scanner := bufio.NewScanner(bytes.NewBuffer(data)) + for scanner.Scan() { + t := scanner.Text() + if strings.HasPrefix(t, "pkgpath ") { + t = strings.TrimPrefix(t, "pkgpath ") + t = strings.TrimSuffix(t, ";") + pkgs = append(pkgs, loadPackage(t, &stk)) + } + } + } else { + pkglistbytes, err := readELFNote(shlibpath, "Go\x00\x00", 1) + if err != nil { + fatalf("readELFNote failed: %v", err) + } + scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) + for scanner.Scan() { + t := scanner.Text() + pkgs = append(pkgs, loadPackage(t, &stk)) + } + } + return +} + // action returns the action for applying the given operation (mode) to the package. // depMode is the action to use when building dependencies. +// action never looks for p in a shared library. func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action { - key := cacheKey{mode, p} + return b.action1(mode, depMode, p, false) +} + +// action1 returns the action for applying the given operation (mode) to the package. +// depMode is the action to use when building dependencies. +// action1 will look for p in a shared library if lookshared is true. +func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool) *action { + shlib := "" + if lookshared { + shlib = p.Shlib + } + key := cacheKey{mode, p, shlib} + a := b.actionCache[key] if a != nil { return a } + if shlib != "" { + key2 := cacheKey{modeInstall, nil, shlib} + a = b.actionCache[key2] + if a != nil { + b.actionCache[key] = a + return a + } + pkgs := readpkglist(shlib) + a = b.libaction(filepath.Base(shlib), pkgs, modeInstall, depMode) + b.actionCache[key2] = a + b.actionCache[key] = a + return a + } a = &action{p: p, pkgdir: p.build.PkgRoot} if p.pkgdir != "" { // overrides p.t a.pkgdir = p.pkgdir } - b.actionCache[key] = a for _, p1 := range p.imports { - a.deps = append(a.deps, b.action(depMode, depMode, p1)) + ls := buildLinkshared + // If p1 is part of the same shared library as p, we need the action + // that builds p here, not the shared libary or we get action loops. + if p1.Shlib == p.Shlib { + ls = false + } + a.deps = append(a.deps, b.action1(depMode, depMode, p1, ls)) } // If we are not doing a cross-build, then record the binary we'll @@ -578,18 +866,15 @@ func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action // using cgo, to make sure we do not overwrite the binary while // a package is using it. If this is a cross-build, then the cgo we // are writing is not the cgo we need to use. - - if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace { - if reqStdPkgSrc { - if len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo" { - var stk importStack - p1 := loadPackage("cmd/cgo", &stk) - if p1.Error != nil { - fatalf("load cmd/cgo: %v", p1.Error) - } - a.cgo = b.action(depMode, depMode, p1) - a.deps = append(a.deps, a.cgo) + if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && reqStdPkgSrc { + if (len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo") && !buildLinkshared && buildBuildmode != "shared" { + var stk importStack + p1 := loadPackage("cmd/cgo", &stk) + if p1.Error != nil { + fatalf("load cmd/cgo: %v", p1.Error) } + a.cgo = b.action(depMode, depMode, p1) + a.deps = append(a.deps, a.cgo) } } @@ -629,8 +914,22 @@ func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action switch mode { case modeInstall: a.f = (*builder).install - a.deps = []*action{b.action(modeBuild, depMode, p)} + a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared)} a.target = a.p.target + + // Install header for cgo in c-archive and c-shared modes. + if p.usesCgo() && (buildBuildmode == "c-archive" || buildBuildmode == "c-shared") { + ah := &action{ + p: a.p, + deps: []*action{a.deps[0]}, + f: (*builder).installHeader, + pkgdir: a.pkgdir, + objdir: a.objdir, + target: a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h", + } + a.deps = append(a.deps, ah) + } + case modeBuild: a.f = (*builder).build a.target = a.objpkg @@ -645,6 +944,13 @@ func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action name := "a.out" if p.exeName != "" { name = p.exeName + } else if goos == "darwin" && buildBuildmode == "c-shared" && p.target != "" { + // On OS X, the linker output name gets recorded in the + // shared library's LC_ID_DYLIB load command. + // The code invoking the linker knows to pass only the final + // path element. Arrange that the path element matches what + // we'll install it as; otherwise the library is only loadable as "a.out". + _, name = filepath.Split(p.target) } a.target = a.objdir + filepath.Join("exe", name) + exeSuffix } @@ -653,6 +959,100 @@ func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action return a } +func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode buildMode) *action { + a := &action{} + if mode == modeBuild { + a.f = (*builder).linkShared + a.target = filepath.Join(b.work, libname) + for _, p := range pkgs { + if p.target == "" { + continue + } + a.deps = append(a.deps, b.action(depMode, depMode, p)) + } + } else if mode == modeInstall { + // Currently build mode shared forces external linking mode, and + // external linking mode forces an import of runtime/cgo. So if it + // was not passed on the command line and it is not present in + // another shared library, add it here. + seencgo := false + _, gccgo := buildToolchain.(gccgoToolchain) + if !gccgo { + for _, p := range pkgs { + seencgo = seencgo || (p.Standard && p.ImportPath == "runtime/cgo") + } + if !seencgo { + var stk importStack + p := loadPackage("runtime/cgo", &stk) + if p.Error != nil { + fatalf("load runtime/cgo: %v", p.Error) + } + computeStale(p) + // If runtime/cgo is in another shared library, then that's + // also the shared library that contains runtime, so + // something will depend on it and so runtime/cgo's staleness + // will be checked when processing that library. + if p.Shlib == "" || p.Shlib == libname { + pkgs = append([]*Package{}, pkgs...) + pkgs = append(pkgs, p) + } + } + } + + // Figure out where the library will go. + var libdir string + for _, p := range pkgs { + plibdir := p.build.PkgTargetRoot + if gccgo { + plibdir = filepath.Join(plibdir, "shlibs") + } + if libdir == "" { + libdir = plibdir + } else if libdir != plibdir { + fatalf("multiple roots %s & %s", libdir, plibdir) + } + } + a.target = filepath.Join(libdir, libname) + + // Now we can check whether we need to rebuild it. + stale := false + var built time.Time + if fi, err := os.Stat(a.target); err == nil { + built = fi.ModTime() + } + for _, p := range pkgs { + if p.target == "" { + continue + } + stale = stale || p.Stale + lstat, err := os.Stat(p.target) + if err != nil || lstat.ModTime().After(built) { + stale = true + } + a.deps = append(a.deps, b.action(depMode, depMode, p)) + } + + if stale { + a.f = (*builder).install + buildAction := b.libaction(libname, pkgs, modeBuild, depMode) + a.deps = []*action{buildAction} + for _, p := range pkgs { + if p.target == "" { + continue + } + shlibnameaction := &action{} + shlibnameaction.f = (*builder).installShlibname + shlibnameaction.target = p.target[:len(p.target)-2] + ".shlibname" + a.deps = append(a.deps, shlibnameaction) + shlibnameaction.deps = append(shlibnameaction.deps, buildAction) + } + } + } else { + fatalf("unregonized mode %v", mode) + } + return a +} + // actionList returns the list of actions in the dag rooted at root // as visited in a depth-first post-order traversal. func actionList(root *action) []*action { @@ -673,6 +1073,31 @@ func actionList(root *action) []*action { return all } +// allArchiveActions returns a list of the archive dependencies of root. +// This is needed because if package p depends on package q that is in libr.so, the +// action graph looks like p->libr.so->q and so just scanning through p's +// dependencies does not find the import dir for q. +func allArchiveActions(root *action) []*action { + seen := map[*action]bool{} + r := []*action{} + var walk func(*action) + walk = func(a *action) { + if seen[a] { + return + } + seen[a] = true + if strings.HasSuffix(a.target, ".so") || a == root { + for _, a1 := range a.deps { + walk(a1) + } + } else if strings.HasSuffix(a.target, ".a") { + r = append(r, a) + } + } + walk(root) + return r +} + // do runs the action graph rooted at root. func (b *builder) do(root *action) { // Build list of all actions, assigning depth-first post-order priority. @@ -793,9 +1218,7 @@ func hasString(strings []string, s string) bool { func (b *builder) build(a *action) (err error) { // Return an error if the package has CXX files but it's not using // cgo nor SWIG, since the CXX files can only be processed by cgo - // and SWIG (it's possible to have packages with C files without - // using cgo, they will get compiled with the plan9 C compiler and - // linked with the rest of the package). + // and SWIG. if len(a.p.CXXFiles) > 0 && !a.p.usesCgo() && !a.p.usesSwig() { return fmt.Errorf("can't build package %s because it contains C++ files (%s) but it's not using cgo nor SWIG", a.p.ImportPath, strings.Join(a.p.CXXFiles, ",")) @@ -824,7 +1247,8 @@ func (b *builder) build(a *action) (err error) { } if a.p.Standard && a.p.ImportPath == "runtime" && buildContext.Compiler == "gc" && - !hasString(a.p.HFiles, "zasm_"+buildContext.GOOS+"_"+buildContext.GOARCH+".h") { + (!hasString(a.p.GoFiles, "zgoos_"+buildContext.GOOS+".go") || + !hasString(a.p.GoFiles, "zgoarch_"+buildContext.GOARCH+".go")) { return fmt.Errorf("%s/%s must be bootstrapped using make%v", buildContext.GOOS, buildContext.GOARCH, defaultSuffix()) } @@ -842,19 +1266,35 @@ func (b *builder) build(a *action) (err error) { } } - var gofiles, cfiles, sfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string + var gofiles, cgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string gofiles = append(gofiles, a.p.GoFiles...) + cgofiles = append(cgofiles, a.p.CgoFiles...) cfiles = append(cfiles, a.p.CFiles...) sfiles = append(sfiles, a.p.SFiles...) + cxxfiles = append(cxxfiles, a.p.CXXFiles...) if a.p.usesCgo() || a.p.usesSwig() { if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.p); err != nil { return } } + + // Run SWIG on each .swig and .swigcxx file. + // Each run will generate two files, a .go file and a .c or .cxx file. + // The .go file will use import "C" and is to be processed by cgo. + if a.p.usesSwig() { + outGo, outC, outCXX, err := b.swig(a.p, obj, pcCFLAGS) + if err != nil { + return err + } + cgofiles = append(cgofiles, outGo...) + cfiles = append(cfiles, outC...) + cxxfiles = append(cxxfiles, outCXX...) + } + // Run cgo. - if a.p.usesCgo() { + if a.p.usesCgo() || a.p.usesSwig() { // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. // There is one exception: runtime/cgo's job is to bridge the // cgo and non-cgo worlds, so it necessarily has files in both. @@ -883,31 +1323,7 @@ func (b *builder) build(a *action) (err error) { if a.cgo != nil && a.cgo.target != "" { cgoExe = a.cgo.target } - outGo, outObj, err := b.cgo(a.p, cgoExe, obj, pcCFLAGS, pcLDFLAGS, gccfiles, a.p.CXXFiles, a.p.MFiles) - if err != nil { - return err - } - cgoObjects = append(cgoObjects, outObj...) - gofiles = append(gofiles, outGo...) - } - - // Run SWIG. - if a.p.usesSwig() { - // In a package using SWIG, any .c or .s files are - // compiled with gcc. - gccfiles := append(cfiles, sfiles...) - cxxfiles, mfiles := a.p.CXXFiles, a.p.MFiles - cfiles = nil - sfiles = nil - - // Don't build c/c++ files twice if cgo is enabled (mainly for pkg-config). - if a.p.usesCgo() { - cxxfiles = nil - gccfiles = nil - mfiles = nil - } - - outGo, outObj, err := b.swig(a.p, obj, pcCFLAGS, gccfiles, cxxfiles, mfiles) + outGo, outObj, err := b.cgo(a.p, cgoExe, obj, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, cxxfiles, a.p.MFiles) if err != nil { return err } @@ -949,10 +1365,10 @@ func (b *builder) build(a *action) (err error) { } // Prepare Go import path list. - inc := b.includeArgs("-I", a.deps) + inc := b.includeArgs("-I", allArchiveActions(a)) // Compile Go. - ofile, out, err := buildToolchain.gc(b, a.p, a.objpkg, obj, inc, gofiles) + ofile, out, err := buildToolchain.gc(b, a.p, a.objpkg, obj, len(sfiles) > 0, inc, gofiles) if len(out) > 0 { b.showOutput(a.p.Dir, a.p.ImportPath, b.processOutput(out)) if err != nil { @@ -977,29 +1393,24 @@ func (b *builder) build(a *action) (err error) { switch { case strings.HasSuffix(name, _goos_goarch): targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { return err } case strings.HasSuffix(name, _goarch): targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { return err } case strings.HasSuffix(name, _goos): targ := file[:len(name)-len(_goos)] + "_GOOS." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { return err } } } - objExt := archChar - if _, ok := buildToolchain.(gccgoToolchain); ok { - objExt = "o" - } - for _, file := range cfiles { - out := file[:len(file)-len(".c")] + "." + objExt + out := file[:len(file)-len(".c")] + ".o" if err := buildToolchain.cc(b, a.p, obj, obj+out, file); err != nil { return err } @@ -1008,7 +1419,7 @@ func (b *builder) build(a *action) (err error) { // Assemble .s files. for _, file := range sfiles { - out := file[:len(file)-len(".s")] + "." + objExt + out := file[:len(file)-len(".s")] + ".o" if err := buildToolchain.asm(b, a.p, obj, obj+out, file); err != nil { return err } @@ -1018,7 +1429,7 @@ func (b *builder) build(a *action) (err error) { // NOTE(rsc): On Windows, it is critically important that the // gcc-compiled objects (cgoObjects) be listed after the ordinary // objects in the archive. I do not know why this is. - // http://golang.org/issue/2601 + // https://golang.org/issue/2601 objects = append(objects, cgoObjects...) // Add system object files. @@ -1043,7 +1454,7 @@ func (b *builder) build(a *action) (err error) { // linker needs the whole dependency tree. all := actionList(a) all = all[:len(all)-1] // drop a - if err := buildToolchain.ld(b, a.p, a.target, all, a.objpkg, objects); err != nil { + if err := buildToolchain.ld(b, a, a.target, all, a.objpkg, objects); err != nil { return err } } @@ -1079,6 +1490,24 @@ func (b *builder) getPkgConfigFlags(p *Package) (cflags, ldflags []string, err e return } +func (b *builder) installShlibname(a *action) error { + a1 := a.deps[0] + err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0644) + if err != nil { + return err + } + if buildX { + b.showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.target), a.target) + } + return nil +} + +func (b *builder) linkShared(a *action) (err error) { + allactions := actionList(a) + allactions = allactions[:len(allactions)-1] + return buildToolchain.ldShared(b, a.deps, a.target, allactions) +} + // install is the action for installing a single package or executable. func (b *builder) install(a *action) (err error) { defer func() { @@ -1089,7 +1518,11 @@ func (b *builder) install(a *action) (err error) { a1 := a.deps[0] perm := os.FileMode(0644) if a1.link { - perm = 0755 + switch buildBuildmode { + case "c-archive", "c-shared": + default: + perm = 0755 + } } // make target directory @@ -1109,7 +1542,7 @@ func (b *builder) install(a *action) (err error) { defer os.Remove(a1.target) } - return b.moveOrCopyFile(a, a.target, a1.target, perm) + return b.moveOrCopyFile(a, a.target, a1.target, perm, false) } // includeArgs returns the -I or -L directory list for access @@ -1126,6 +1559,9 @@ func (b *builder) includeArgs(flag string, all []*action) []string { // This is the $WORK/my/package/_test directory for the // package being built, so there are few of these. for _, a1 := range all { + if a1.p == nil { + continue + } if dir := a1.pkgdir; dir != a1.p.build.PkgRoot && !incMap[dir] { incMap[dir] = true inc = append(inc, flag, dir) @@ -1138,17 +1574,12 @@ func (b *builder) includeArgs(flag string, all []*action) []string { // Finally, look in the installed package directories for each action. for _, a1 := range all { + if a1.p == nil { + continue + } if dir := a1.pkgdir; dir == a1.p.build.PkgRoot && !incMap[dir] { incMap[dir] = true - if _, ok := buildToolchain.(gccgoToolchain); ok { - dir = filepath.Join(dir, "gccgo_"+goos+"_"+goarch) - } else { - dir = filepath.Join(dir, goos+"_"+goarch) - if buildContext.InstallSuffix != "" { - dir += "_" + buildContext.InstallSuffix - } - } - inc = append(inc, flag, dir) + inc = append(inc, flag, a1.p.build.PkgTargetRoot) } } @@ -1156,7 +1587,7 @@ func (b *builder) includeArgs(flag string, all []*action) []string { } // moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode) error { +func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode, force bool) error { if buildN { b.showcmd("", "mv %s %s", src, dst) return nil @@ -1173,11 +1604,11 @@ func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode) e } } - return b.copyFile(a, dst, src, perm) + return b.copyFile(a, dst, src, perm, force) } // copyFile is like 'cp src dst'. -func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode) error { +func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force bool) error { if buildN || buildX { b.showcmd("", "cp %s %s", src, dst) if buildN { @@ -1198,7 +1629,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode) error { if fi.IsDir() { return fmt.Errorf("build output %q already exists and is a directory", dst) } - if !isObject(dst) { + if !force && !isObject(dst) { return fmt.Errorf("build output %q already exists and is not an object file", dst) } } @@ -1235,10 +1666,30 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode) error { return nil } +// Install the cgo export header file, if there is one. +func (b *builder) installHeader(a *action) error { + src := a.objdir + "_cgo_install.h" + if _, err := os.Stat(src); os.IsNotExist(err) { + // If the file does not exist, there are no exported + // functions, and we do not install anything. + return nil + } + + dir, _ := filepath.Split(a.target) + if dir != "" { + if err := b.mkdir(dir); err != nil { + return err + } + } + + return b.moveOrCopyFile(a, a.target, src, 0644, true) +} + // cover runs, in effect, // go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go func (b *builder) cover(a *action, dst, src string, perm os.FileMode, varName string) error { return b.run(a.objdir, "cover "+a.p.ImportPath, nil, + buildToolExec, tool("cover"), "-mode", a.p.coverMode, "-var", varName, @@ -1247,15 +1698,15 @@ func (b *builder) cover(a *action, dst, src string, perm os.FileMode, varName st } var objectMagic = [][]byte{ - {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive - {'\x7F', 'E', 'L', 'F'}, // ELF - {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit - {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit - {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit - {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit - {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x04, 0x00}, // PE (Windows) as generated by 6l/8l - {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 - {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive + {'\x7F', 'E', 'L', 'F'}, // ELF + {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit + {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit + {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit + {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit + {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc + {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 + {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 } func isObject(s string) bool { @@ -1436,7 +1887,7 @@ func (b *builder) runOut(dir string, desc string, env []string, cmdargs ...inter cmd.Stdout = &buf cmd.Stderr = &buf cmd.Dir = dir - cmd.Env = mergeEnvLists(env, envForDir(cmd.Dir)) + cmd.Env = mergeEnvLists(env, envForDir(cmd.Dir, os.Environ())) err := cmd.Run() // cmd.Run will fail on Unix if some other process has the binary @@ -1478,7 +1929,7 @@ func (b *builder) runOut(dir string, desc string, env []string, cmdargs ...inter // Sleeping when we observe the race seems to be the most reliable // option we have. // - // http://golang.org/issue/3001 + // https://golang.org/issue/3001 // if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { time.Sleep(100 * time.Millisecond << uint(nbusy)) @@ -1561,7 +2012,7 @@ type toolchain interface { // gc runs the compiler in a specific directory on a set of files // and returns the name of the generated output file. // The compiler runs in the directory dir. - gc(b *builder, p *Package, archive, obj string, importArgs []string, gofiles []string) (ofile string, out []byte, err error) + gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) // cc runs the toolchain's C compiler in a directory on a C file // to produce an output file. cc(b *builder, p *Package, objdir, ofile, cfile string) error @@ -1574,8 +2025,10 @@ type toolchain interface { // an archive from a set of object files. // typically it is run in the object directory. pack(b *builder, p *Package, objDir, afile string, ofiles []string) error - // ld runs the linker to create a package starting at mainpkg. - ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error + // ld runs the linker to create an executable starting at mainpkg. + ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error + // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions + ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error compiler() string linker() string @@ -1598,7 +2051,7 @@ func (noToolchain) linker() string { return "" } -func (noToolchain) gc(b *builder, p *Package, archive, obj string, importArgs []string, gofiles []string) (ofile string, out []byte, err error) { +func (noToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) { return "", nil, noCompiler() } @@ -1615,7 +2068,11 @@ func (noToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s return noCompiler() } -func (noToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { +func (noToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { return noCompiler() } @@ -1627,24 +2084,27 @@ func (noToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error type gcToolchain struct{} func (gcToolchain) compiler() string { - return tool(archChar + "g") + return tool("compile") } func (gcToolchain) linker() string { - return tool(archChar + "l") + return tool("link") } -func (gcToolchain) gc(b *builder, p *Package, archive, obj string, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { +func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { if archive != "" { ofile = archive } else { - out := "_go_." + archChar + out := "_go_.o" ofile = obj + out } gcargs := []string{"-p", p.ImportPath} + if p.Name == "main" { + gcargs[1] = "main" + } if p.Standard && p.ImportPath == "runtime" { - // runtime compiles with a special 6g flag to emit + // runtime compiles with a special gc flag to emit // additional reflect type data. gcargs = append(gcargs, "-+") } @@ -1666,24 +2126,76 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, importArgs [] if buildContext.InstallSuffix != "" { gcargs = append(gcargs, "-installsuffix", buildContext.InstallSuffix) } + if p.buildID != "" { + gcargs = append(gcargs, "-buildid", p.buildID) + } + + for _, path := range p.Imports { + if i := strings.LastIndex(path, "/vendor/"); i >= 0 { + gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path) + } else if strings.HasPrefix(path, "vendor/") { + gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path) + } + } + + for _, path := range p.Imports { + if i := strings.LastIndex(path, "/vendor/"); i >= 0 { + gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path) + } else if strings.HasPrefix(path, "vendor/") { + gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path) + } + } - args := stringList(tool(archChar+"g"), "-o", ofile, "-trimpath", b.work, buildGcflags, gcargs, "-D", p.localPrefix, importArgs) + args := []interface{}{buildToolExec, tool("compile"), "-o", ofile, "-trimpath", b.work, buildGcflags, gcargs, "-D", p.localPrefix, importArgs} if ofile == archive { args = append(args, "-pack") } + if asmhdr { + args = append(args, "-asmhdr", obj+"go_asm.h") + } for _, f := range gofiles { args = append(args, mkAbs(p.Dir, f)) } - output, err = b.runOut(p.Dir, p.ImportPath, nil, args) + output, err = b.runOut(p.Dir, p.ImportPath, nil, args...) return ofile, output, err } func (gcToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. - inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) + inc := filepath.Join(goroot, "pkg", "include") sfile = mkAbs(p.Dir, sfile) - return b.run(p.Dir, p.ImportPath, nil, tool(archChar+"a"), "-trimpath", b.work, "-I", obj, "-I", inc, "-o", ofile, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, sfile) + args := []interface{}{buildToolExec, tool("asm"), "-o", ofile, "-trimpath", b.work, "-I", obj, "-I", inc, "-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch, buildAsmflags, sfile} + if err := b.run(p.Dir, p.ImportPath, nil, args...); err != nil { + return err + } + return nil +} + +// toolVerify checks that the command line args writes the same output file +// if run using newTool instead. +// Unused now but kept around for future use. +func toolVerify(b *builder, p *Package, newTool string, ofile string, args []interface{}) error { + newArgs := make([]interface{}, len(args)) + copy(newArgs, args) + newArgs[1] = tool(newTool) + newArgs[3] = ofile + ".new" // x.6 becomes x.6.new + if err := b.run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { + return err + } + data1, err := ioutil.ReadFile(ofile) + if err != nil { + return err + } + data2, err := ioutil.ReadFile(ofile + ".new") + if err != nil { + return err + } + if !bytes.Equal(data1, data2) { + return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(stringList(args...), " "), strings.Join(stringList(newArgs...), " ")) + } + os.Remove(ofile + ".new") + return nil } func (gcToolchain) pkgpath(basedir string, p *Package) string { @@ -1722,7 +2234,7 @@ func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s // Need actual pack. cmdline[0] = tool("pack") - return b.run(p.Dir, p.ImportPath, nil, cmdline) + return b.run(p.Dir, p.ImportPath, nil, buildToolExec, cmdline) } func packInternal(b *builder, afile string, ofiles []string) error { @@ -1775,21 +2287,49 @@ func packInternal(b *builder, afile string, ofiles []string) error { return dst.Close() } -func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { +// setextld sets the appropriate linker flags for the specified compiler. +func setextld(ldflags []string, compiler []string) []string { + for _, f := range ldflags { + if f == "-extld" || strings.HasPrefix(f, "-extld=") { + // don't override -extld if supplied + return ldflags + } + } + ldflags = append(ldflags, "-extld="+compiler[0]) + if len(compiler) > 1 { + extldflags := false + add := strings.Join(compiler[1:], " ") + for i, f := range ldflags { + if f == "-extldflags" && i+1 < len(ldflags) { + ldflags[i+1] = add + " " + ldflags[i+1] + extldflags = true + break + } else if strings.HasPrefix(f, "-extldflags=") { + ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] + extldflags = true + break + } + } + if !extldflags { + ldflags = append(ldflags, "-extldflags="+add) + } + } + return ldflags +} + +func (gcToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { importArgs := b.includeArgs("-L", allactions) - cxx := len(p.CXXFiles) > 0 + cxx := len(root.p.CXXFiles) > 0 || len(root.p.SwigCXXFiles) > 0 for _, a := range allactions { - if a.p != nil && len(a.p.CXXFiles) > 0 { + if a.p != nil && (len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0) { cxx = true } } - ldflags := buildLdflags - // Limit slice capacity so that concurrent appends do not race on the shared array. - ldflags = ldflags[:len(ldflags):len(ldflags)] + var ldflags []string if buildContext.InstallSuffix != "" { ldflags = append(ldflags, "-installsuffix", buildContext.InstallSuffix) } - if p.omitDWARF { + if root.p.omitDWARF { ldflags = append(ldflags, "-w") } @@ -1797,56 +2337,67 @@ func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, // appropriate linker. In case of C++ code, use the compiler named // by the CXX environment variable or defaultCXX if CXX is not set. // Else, use the CC environment variable and defaultCC as fallback. - extld := false - for _, f := range ldflags { - if f == "-extld" || strings.HasPrefix(f, "-extld=") { - extld = true - break + var compiler []string + if cxx { + compiler = envList("CXX", defaultCXX) + } else { + compiler = envList("CC", defaultCC) + } + ldflags = setextld(ldflags, compiler) + ldflags = append(ldflags, "-buildmode="+ldBuildmode) + if root.p.buildID != "" { + ldflags = append(ldflags, "-buildid="+root.p.buildID) + } + ldflags = append(ldflags, buildLdflags...) + + // On OS X when using external linking to build a shared library, + // the argument passed here to -o ends up recorded in the final + // shared library in the LC_ID_DYLIB load command. + // To avoid putting the temporary output directory name there + // (and making the resulting shared library useless), + // run the link in the output directory so that -o can name + // just the final path element. + dir := "." + if goos == "darwin" && buildBuildmode == "c-shared" { + dir, out = filepath.Split(out) + } + + return b.run(dir, root.p.ImportPath, nil, buildToolExec, tool("link"), "-o", out, importArgs, ldflags, mainpkg) +} + +func (gcToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { + importArgs := b.includeArgs("-L", allactions) + ldflags := []string{"-installsuffix", buildContext.InstallSuffix} + ldflags = append(ldflags, "-buildmode=shared") + ldflags = append(ldflags, buildLdflags...) + cxx := false + for _, a := range allactions { + if a.p != nil && (len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0) { + cxx = true } } - if !extld { - var compiler []string - if cxx { - compiler = envList("CXX", defaultCXX) - } else { - compiler = envList("CC", defaultCC) - } - ldflags = append(ldflags, "-extld="+compiler[0]) - if len(compiler) > 1 { - extldflags := false - add := strings.Join(compiler[1:], " ") - for i, f := range ldflags { - if f == "-extldflags" && i+1 < len(ldflags) { - ldflags[i+1] = add + " " + ldflags[i+1] - extldflags = true - break - } else if strings.HasPrefix(f, "-extldflags=") { - ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] - extldflags = true - break - } - } - if !extldflags { - ldflags = append(ldflags, "-extldflags="+add) - } + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", defaultCXX) + } else { + compiler = envList("CC", defaultCC) + } + ldflags = setextld(ldflags, compiler) + for _, d := range toplevelactions { + if !strings.HasSuffix(d.target, ".a") { // omit unsafe etc and actions for other shared libraries + continue } + ldflags = append(ldflags, d.p.ImportPath+"="+d.target) } - return b.run(".", p.ImportPath, nil, tool(archChar+"l"), "-o", out, importArgs, ldflags, mainpkg) + return b.run(".", out, nil, buildToolExec, tool("link"), "-o", out, importArgs, ldflags) } func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { - inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) - cfile = mkAbs(p.Dir, cfile) - warn := []string{"-w"} - if p.usesSwig() { - // When using SWIG, this compiler is only used to - // compile the C files generated by SWIG. - // We don't want warnings. - // See issue 9065 for details. - warn = nil - } - args := stringList(tool(archChar+"c"), "-F", "-V", warn, "-trimpath", b.work, "-I", objdir, "-I", inc, "-o", ofile, buildCcflags, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, cfile) - return b.run(p.Dir, p.ImportPath, nil, args) + return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(p.Dir, cfile)) } // The Gccgo toolchain. @@ -1870,7 +2421,7 @@ func (gccgoToolchain) linker() string { return gccgoBin } -func (tools gccgoToolchain) gc(b *builder, p *Package, archive, obj string, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { +func (tools gccgoToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { out := "_go_.o" ofile = obj + out gcargs := []string{"-g"} @@ -1896,8 +2447,8 @@ func (tools gccgoToolchain) asm(b *builder, p *Package, obj, ofile, sfile string if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) } + defs = tools.maybePIC(defs) defs = append(defs, b.gccArchArgs()...) - return b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-c", "-I", obj, "-o", ofile, defs, sfile) } @@ -1916,30 +2467,46 @@ func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles return b.run(p.Dir, p.ImportPath, nil, "ar", "cru", mkAbs(objDir, afile), absOfiles) } -func (tools gccgoToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { +func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { // gccgo needs explicit linking with all package dependencies, // and all LDFLAGS from cgo dependencies. apackagesSeen := make(map[*Package]bool) afiles := []string{} + shlibs := []string{} xfiles := []string{} ldflags := b.gccArchArgs() cgoldflags := []string{} usesCgo := false - cxx := len(p.CXXFiles) > 0 - objc := len(p.MFiles) > 0 - - // Prefer the output of an install action to the output of a build action, - // because the install action will delete the output of the build action. - // Iterate over the list backward (reverse dependency order) so that we - // always see the install before the build. - for i := len(allactions) - 1; i >= 0; i-- { - a := allactions[i] - if !a.p.Standard { - if a.p != nil && !apackagesSeen[a.p] { + cxx := len(root.p.CXXFiles) > 0 || len(root.p.SwigCXXFiles) > 0 + objc := len(root.p.MFiles) > 0 + + actionsSeen := make(map[*action]bool) + // Make a pre-order depth-first traversal of the action graph, taking note of + // whether a shared library action has been seen on the way to an action (the + // construction of the graph means that if any path to a node passes through + // a shared library action, they all do). + var walk func(a *action, seenShlib bool) + walk = func(a *action, seenShlib bool) { + if actionsSeen[a] { + return + } + actionsSeen[a] = true + if a.p != nil && !seenShlib { + if a.p.Standard { + return + } + // We record the target of the first time we see a .a file + // for a package to make sure that we prefer the 'install' + // rather than the 'build' location (which may not exist any + // more). We still need to traverse the dependencies of the + // build action though so saying + // if apackagesSeen[a.p] { return } + // doesn't work. + if !apackagesSeen[a.p] { apackagesSeen[a.p] = true if a.p.fake && a.p.external { // external _tests, if present must come before - // internal _tests. Store these on a seperate list + // internal _tests. Store these on a separate list // and place them at the head after this loop. xfiles = append(xfiles, a.target) } else if a.p.fake { @@ -1950,54 +2517,182 @@ func (tools gccgoToolchain) ld(b *builder, p *Package, out string, allactions [] } } } + if strings.HasSuffix(a.target, ".so") { + shlibs = append(shlibs, a.target) + seenShlib = true + } + for _, a1 := range a.deps { + walk(a1, seenShlib) + } + } + for _, a1 := range root.deps { + walk(a1, false) } afiles = append(xfiles, afiles...) for _, a := range allactions { - if a.p != nil { + // Gather CgoLDFLAGS, but not from standard packages. + // The go tool can dig up runtime/cgo from GOROOT and + // think that it should use its CgoLDFLAGS, but gccgo + // doesn't use runtime/cgo. + if a.p == nil { + continue + } + if !a.p.Standard { cgoldflags = append(cgoldflags, a.p.CgoLDFLAGS...) - if len(a.p.CgoFiles) > 0 { - usesCgo = true - } - if a.p.usesSwig() { - usesCgo = true - } - if len(a.p.CXXFiles) > 0 { - cxx = true - } - if len(a.p.MFiles) > 0 { - objc = true - } + } + if len(a.p.CgoFiles) > 0 { + usesCgo = true + } + if a.p.usesSwig() { + usesCgo = true + } + if len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0 { + cxx = true + } + if len(a.p.MFiles) > 0 { + objc = true } } + + switch ldBuildmode { + case "c-archive", "c-shared": + ldflags = append(ldflags, "-Wl,--whole-archive") + } + ldflags = append(ldflags, afiles...) + + switch ldBuildmode { + case "c-archive", "c-shared": + ldflags = append(ldflags, "-Wl,--no-whole-archive") + } + ldflags = append(ldflags, cgoldflags...) ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) - ldflags = append(ldflags, p.CgoLDFLAGS...) - if usesCgo && goos == "linux" { - ldflags = append(ldflags, "-Wl,-E") + ldflags = append(ldflags, root.p.CgoLDFLAGS...) + + ldflags = stringList("-Wl,-(", ldflags, "-Wl,-)") + + for _, shlib := range shlibs { + ldflags = append( + ldflags, + "-L"+filepath.Dir(shlib), + "-Wl,-rpath="+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) } - if cxx { - ldflags = append(ldflags, "-lstdc++") + + var realOut string + switch ldBuildmode { + case "exe": + if usesCgo && goos == "linux" { + ldflags = append(ldflags, "-Wl,-E") + } + + case "c-archive": + // Link the Go files into a single .o, and also link + // in -lgolibbegin. + // + // We need to use --whole-archive with -lgolibbegin + // because it doesn't define any symbols that will + // cause the contents to be pulled in; it's just + // initialization code. + // + // The user remains responsible for linking against + // -lgo -lpthread -lm in the final link. We can't use + // -r to pick them up because we can't combine + // split-stack and non-split-stack code in a single -r + // link, and libgo picks up non-split-stack code from + // libffi. + ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") + + // We are creating an object file, so we don't want a build ID. + ldflags = b.disableBuildID(ldflags) + + realOut = out + out = out + ".o" + + case "c-shared": + ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc") + + default: + fatalf("-buildmode=%s not supported for gccgo", ldBuildmode) + } + + switch ldBuildmode { + case "exe", "c-shared": + if cxx { + ldflags = append(ldflags, "-lstdc++") + } + if objc { + ldflags = append(ldflags, "-lobjc") + } } - if objc { - ldflags = append(ldflags, "-lobjc") + + if err := b.run(".", root.p.ImportPath, nil, tools.linker(), "-o", out, ofiles, ldflags, buildGccgoflags); err != nil { + return err + } + + switch ldBuildmode { + case "c-archive": + if err := b.run(".", root.p.ImportPath, nil, "ar", "rc", realOut, out); err != nil { + return err + } } - return b.run(".", p.ImportPath, nil, tools.linker(), "-o", out, ofiles, "-Wl,-(", ldflags, "-Wl,-)", buildGccgoflags) + return nil } -func (gccgoToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { - inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) +func (tools gccgoToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { + args := []string{"-o", out, "-shared", "-nostdlib", "-zdefs", "-Wl,--whole-archive"} + for _, a := range toplevelactions { + args = append(args, a.target) + } + args = append(args, "-Wl,--no-whole-archive", "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") + shlibs := []string{} + for _, a := range allactions { + if strings.HasSuffix(a.target, ".so") { + shlibs = append(shlibs, a.target) + } + } + for _, shlib := range shlibs { + args = append( + args, + "-L"+filepath.Dir(shlib), + "-Wl,-rpath="+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) + } + return b.run(".", out, nil, tools.linker(), args, buildGccgoflags) +} + +func (tools gccgoToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { + inc := filepath.Join(goroot, "pkg", "include") cfile = mkAbs(p.Dir, cfile) defs := []string{"-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch} defs = append(defs, b.gccArchArgs()...) if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) } + switch goarch { + case "386", "amd64": + defs = append(defs, "-fsplit-stack") + } + defs = tools.maybePIC(defs) return b.run(p.Dir, p.ImportPath, nil, envList("CC", defaultCC), "-Wall", "-g", "-I", objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) } +// maybePIC adds -fPIC to the list of arguments if needed. +func (tools gccgoToolchain) maybePIC(args []string) []string { + switch buildBuildmode { + case "c-shared", "shared": + args = append(args, "-fPIC") + } + return args +} + func gccgoPkgpath(p *Package) string { if p.build.IsCommand() && !p.forceLibrary { return "" @@ -2073,7 +2768,7 @@ func (b *builder) ccompile(p *Package, out string, flags []string, file string, // gccld runs the gcc linker to create an executable from a set of object files. func (b *builder) gccld(p *Package, out string, flags []string, obj []string) error { var cmd []string - if len(p.CXXFiles) > 0 { + if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { cmd = b.gxxCmd(p.Dir) } else { cmd = b.gccCmd(p.Dir) @@ -2132,7 +2827,7 @@ func (b *builder) ccompilerCmd(envvar, defcmd, objdir string) []string { // On OS X, some of the compilers behave as if -fno-common // is always set, and the Mach-O linker in 6l/8l assumes this. - // See http://golang.org/issue/3253. + // See https://golang.org/issue/3253. if goos == "darwin" { a = append(a, "-fno-common") } @@ -2142,12 +2837,12 @@ func (b *builder) ccompilerCmd(envvar, defcmd, objdir string) []string { // gccArchArgs returns arguments to pass to gcc based on the architecture. func (b *builder) gccArchArgs() []string { - switch archChar { - case "8": + switch goarch { + case "386": return []string{"-m32"} - case "6": + case "amd64", "amd64p32": return []string{"-m64"} - case "5": + case "arm": return []string{"-marm"} // not thumb } return nil @@ -2185,7 +2880,7 @@ var ( cgoLibGccFileOnce sync.Once ) -func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfiles, gxxfiles, mfiles []string) (outGo, outObj []string, err error) { +func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles []string) (outGo, outObj []string, err error) { cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoLDFLAGS := b.cflags(p, true) _, cgoexeCFLAGS, _, _ := b.cflags(p, false) cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) @@ -2202,7 +2897,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi // TODO: CGOPKGPATH, CGO_FLAGS? gofiles := []string{obj + "_cgo_gotypes.go"} cfiles := []string{"_cgo_main.c", "_cgo_export.c"} - for _, fn := range p.CgoFiles { + for _, fn := range cgofiles { f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") gofiles = append(gofiles, obj+f+"cgo1.go") cfiles = append(cfiles, f+"cgo2.c") @@ -2212,8 +2907,6 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi cgoflags := []string{} // TODO: make cgo not depend on $GOARCH? - objExt := archChar - if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = append(cgoflags, "-import_runtime_cgo=false") } @@ -2232,23 +2925,38 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi } if _, ok := buildToolchain.(gccgoToolchain); ok { + switch goarch { + case "386", "amd64": + cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") + } cgoflags = append(cgoflags, "-gccgo") if pkgpath := gccgoPkgpath(p); pkgpath != "" { cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) } - objExt = "o" } - if err := b.run(p.Dir, p.ImportPath, cgoenv, cgoExe, "-objdir", obj, cgoflags, "--", cgoCPPFLAGS, cgoexeCFLAGS, p.CgoFiles); err != nil { + + switch buildBuildmode { + case "c-archive", "c-shared": + // Tell cgo that if there are any exported functions + // it should generate a header file that C code can + // #include. + cgoflags = append(cgoflags, "-exportheader="+obj+"_cgo_install.h") + } + + if err := b.run(p.Dir, p.ImportPath, cgoenv, buildToolExec, cgoExe, "-objdir", obj, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoexeCFLAGS, cgofiles); err != nil { return nil, nil, err } outGo = append(outGo, gofiles...) // cc _cgo_defun.c - defunObj := obj + "_cgo_defun." + objExt - if err := buildToolchain.cc(b, p, obj, defunObj, defunC); err != nil { - return nil, nil, err + _, gccgo := buildToolchain.(gccgoToolchain) + if gccgo { + defunObj := obj + "_cgo_defun.o" + if err := buildToolchain.cc(b, p, obj, defunObj, defunC); err != nil { + return nil, nil, err + } + outObj = append(outObj, defunObj) } - outObj = append(outObj, defunObj) // gcc var linkobj []string @@ -2362,20 +3070,15 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi } // cgo -dynimport - importC := obj + "_cgo_import.c" + importGo := obj + "_cgo_import.go" cgoflags = []string{} if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = append(cgoflags, "-dynlinker") // record path to dynamic linker } - if err := b.run(p.Dir, p.ImportPath, nil, cgoExe, "-objdir", obj, "-dynimport", dynobj, "-dynout", importC, cgoflags); err != nil { - return nil, nil, err - } - - // cc _cgo_import.ARCH - importObj := obj + "_cgo_import." + objExt - if err := buildToolchain.cc(b, p, obj, importObj, importC); err != nil { + if err := b.run(p.Dir, p.ImportPath, nil, buildToolExec, cgoExe, "-objdir", obj, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags); err != nil { return nil, nil, err } + outGo = append(outGo, importGo) ofile := obj + "_all.o" var gccObjs, nonGccObjs []string @@ -2388,19 +3091,8 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi } ldflags := stringList(bareLDFLAGS, "-Wl,-r", "-nostdlib", staticLibs) - // Some systems, such as Ubuntu, always add --build-id to - // every link, but we don't want a build ID since we are - // producing an object file. On some of those system a plain - // -r (not -Wl,-r) will turn off --build-id, but clang 3.0 - // doesn't support a plain -r. I don't know how to turn off - // --build-id when using clang other than passing a trailing - // --build-id=none. So that is what we do, but only on - // systems likely to support it, which is to say, systems that - // normally use gold or the GNU linker. - switch goos { - case "android", "dragonfly", "linux", "netbsd": - ldflags = append(ldflags, "-Wl,--build-id=none") - } + // We are creating an object file, so we don't want a build ID. + ldflags = b.disableBuildID(ldflags) if err := b.gccld(p, ofile, ldflags, gccObjs); err != nil { return nil, nil, err @@ -2408,8 +3100,8 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi // NOTE(rsc): The importObj is a 5c/6c/8c object and on Windows // must be processed before the gcc-generated objects. - // Put it first. http://golang.org/issue/2601 - outObj = stringList(importObj, nonGccObjs, ofile) + // Put it first. https://golang.org/issue/2601 + outObj = stringList(nonGccObjs, ofile) return outGo, outObj, nil } @@ -2417,77 +3109,41 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, gccfi // Run SWIG on all SWIG input files. // TODO: Don't build a shared library, once SWIG emits the necessary // pragmas for external linking. -func (b *builder) swig(p *Package, obj string, pcCFLAGS, gccfiles, gxxfiles, mfiles []string) (outGo, outObj []string, err error) { - cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _ := b.cflags(p, true) - cflags := stringList(cgoCPPFLAGS, cgoCFLAGS) - cxxflags := stringList(cgoCPPFLAGS, cgoCXXFLAGS) - - for _, file := range gccfiles { - ofile := obj + cgoRe.ReplaceAllString(file[:len(file)-1], "_") + "o" - if err := b.gcc(p, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range gxxfiles { - // Append .o to the file, just in case the pkg has file.c and file.cpp - ofile := obj + cgoRe.ReplaceAllString(file, "_") + ".o" - if err := b.gxx(p, ofile, cxxflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range mfiles { - // Append .o to the file, just in case the pkg has file.c and file.cpp - ofile := obj + cgoRe.ReplaceAllString(file, "_") + ".o" - if err := b.gcc(p, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - +func (b *builder) swig(p *Package, obj string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { if err := b.swigVersionCheck(); err != nil { - return nil, nil, err + return nil, nil, nil, err } intgosize, err := b.swigIntSize(obj) if err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, f := range p.SwigFiles { - goFile, objFile, gccObjFile, err := b.swigOne(p, f, obj, pcCFLAGS, false, intgosize) + goFile, cFile, err := b.swigOne(p, f, obj, pcCFLAGS, false, intgosize) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if goFile != "" { outGo = append(outGo, goFile) } - if objFile != "" { - outObj = append(outObj, objFile) - } - if gccObjFile != "" { - outObj = append(outObj, gccObjFile) + if cFile != "" { + outC = append(outC, cFile) } } for _, f := range p.SwigCXXFiles { - goFile, objFile, gccObjFile, err := b.swigOne(p, f, obj, pcCFLAGS, true, intgosize) + goFile, cxxFile, err := b.swigOne(p, f, obj, pcCFLAGS, true, intgosize) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if goFile != "" { outGo = append(outGo, goFile) } - if objFile != "" { - outObj = append(outObj, objFile) - } - if gccObjFile != "" { - outObj = append(outObj, gccObjFile) + if cxxFile != "" { + outCXX = append(outCXX, cxxFile) } } - return outGo, outObj, nil + return outGo, outC, outCXX, nil } // Make sure SWIG is new enough. @@ -2501,20 +3157,51 @@ func (b *builder) swigDoVersionCheck() error { if err != nil { return err } - re := regexp.MustCompile(`[vV]ersion +([\d])`) + re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) matches := re.FindSubmatch(out) if matches == nil { // Can't find version number; hope for the best. return nil } + major, err := strconv.Atoi(string(matches[1])) if err != nil { // Can't find version number; hope for the best. return nil } + const errmsg = "must have SWIG version >= 3.0.6" if major < 3 { - return errors.New("must have SWIG version >= 3.0") + return errors.New(errmsg) + } + if major > 3 { + // 4.0 or later + return nil + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return nil + } + if minor > 0 { + // 3.1 or later + return nil + } } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return nil + } + if patch < 6 { + // Before 3.0.6. + return errors.New(errmsg) + } + } + return nil } @@ -2545,14 +3232,14 @@ func (b *builder) swigIntSize(obj string) (intsize string, err error) { p := goFilesPackage(srcs) - if _, _, e := buildToolchain.gc(b, p, "", obj, nil, srcs); e != nil { + if _, _, e := buildToolchain.gc(b, p, "", obj, false, nil, srcs); e != nil { return "32", nil } return "64", nil } // Run SWIG on one SWIG input file. -func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outObj, objGccObj string, err error) { +func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _ := b.cflags(p, true) var cflags []string if cxx { @@ -2567,7 +3254,6 @@ func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx b } base := file[:len(file)-n] goFile := base + ".go" - cBase := base + "_gc." gccBase := base + "_wrap." gccExt := "c" if cxx { @@ -2579,6 +3265,7 @@ func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx b // swig args := []string{ "-go", + "-cgo", "-intgosize", intgosize, "-module", base, "-o", obj + gccBase + gccExt, @@ -2601,39 +3288,40 @@ func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx b args = append(args, "-c++") } - if out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file); err != nil { + out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file) + if err != nil { if len(out) > 0 { - if bytes.Contains(out, []byte("Unrecognized option -intgosize")) { - return "", "", "", errors.New("must have SWIG version >= 3.0") + if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { + return "", "", errors.New("must have SWIG version >= 3.0.6") } - b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) - return "", "", "", errPrintedOutput + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig error + return "", "", errPrintedOutput } - return "", "", "", err + return "", "", err } - - var cObj string - if !gccgo { - // cc - cObj = obj + cBase + archChar - if err := buildToolchain.cc(b, p, obj, cObj, obj+cBase+"c"); err != nil { - return "", "", "", err - } + if len(out) > 0 { + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig warning } - // gcc - gccObj := obj + gccBase + "o" - if !cxx { - if err := b.gcc(p, gccObj, cflags, obj+gccBase+gccExt); err != nil { - return "", "", "", err - } - } else { - if err := b.gxx(p, gccObj, cflags, obj+gccBase+gccExt); err != nil { - return "", "", "", err - } - } + return obj + goFile, obj + gccBase + gccExt, nil +} - return obj + goFile, cObj, gccObj, nil +// disableBuildID adjusts a linker command line to avoid creating a +// build ID when creating an object file rather than an executable or +// shared library. Some systems, such as Ubuntu, always add +// --build-id to every link, but we don't want a build ID when we are +// producing an object file. On some of those system a plain -r (not +// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a +// plain -r. I don't know how to turn off --build-id when using clang +// other than passing a trailing --build-id=none. So that is what we +// do, but only on systems likely to support it, which is to say, +// systems that normally use gold or the GNU linker. +func (b *builder) disableBuildID(ldflags []string) []string { + switch goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, "-Wl,--build-id=none") + } + return ldflags } // An actionQueue is a priority queue of actions. @@ -2669,7 +3357,6 @@ func raceInit() { } buildGcflags = append(buildGcflags, "-race") buildLdflags = append(buildLdflags, "-race") - buildCcflags = append(buildCcflags, "-D", "RACE") if buildContext.InstallSuffix != "" { buildContext.InstallSuffix += "_" } diff --git a/libgo/go/cmd/go/doc.go b/libgo/go/cmd/go/doc.go index 7191ee0f3a9..4a07dfe11f4 100644 --- a/libgo/go/cmd/go/doc.go +++ b/libgo/go/cmd/go/doc.go @@ -1,1131 +1,103 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// DO NOT EDIT THIS FILE. GENERATED BY mkdoc.sh. -// Edit the documentation in other files and rerun mkdoc.sh to generate this one. - -/* -Go is a tool for managing Go source code. - -Usage: - - go command [arguments] - -The commands are: - - build compile packages and dependencies - clean remove object files - env print Go environment information - fix run go tool fix on packages - fmt run gofmt on package sources - generate generate Go files by processing source - get download and install packages and dependencies - install compile and install packages and dependencies - list list packages - run compile and run Go program - test test packages - tool run specified go tool - version print Go version - vet run go tool vet on packages - -Use "go help [command]" for more information about a command. - -Additional help topics: - - c calling between Go and C - filetype file types - gopath GOPATH environment variable - importpath import path syntax - packages description of package lists - testflag description of testing flags - testfunc description of testing functions - -Use "go help [topic]" for more information about that topic. - - -Compile packages and dependencies - -Usage: - - go build [-o output] [-i] [build flags] [packages] - -Build compiles the packages named by the import paths, -along with their dependencies, but it does not install the results. - -If the arguments are a list of .go files, build treats them as a list -of source files specifying a single package. - -When the command line specifies a single main package, -build writes the resulting executable to output. -Otherwise build compiles the packages but discards the results, -serving only as a check that the packages can be built. - -The -o flag specifies the output file name. If not specified, the -output file name depends on the arguments and derives from the name -of the package, such as p.a for package p, unless p is 'main'. If -the package is main and file names are provided, the file name -derives from the first file name mentioned, such as f1 for 'go build -f1.go f2.go'; with no files provided ('go build'), the output file -name is the base name of the containing directory. - -The -i flag installs the packages that are dependencies of the target. - -The build flags are shared by the build, clean, get, install, list, run, -and test commands: - - -a - force rebuilding of packages that are already up-to-date. - In Go releases, does not apply to the standard library. - -n - print the commands but do not run them. - -p n - the number of builds that can be run in parallel. - The default is the number of CPUs available. - -race - enable data race detection. - Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. - -v - print the names of packages as they are compiled. - -work - print the name of the temporary work directory and - do not delete it when exiting. - -x - print the commands. - - -ccflags 'arg list' - arguments to pass on each 5c, 6c, or 8c compiler invocation. - -compiler name - name of compiler to use, as in runtime.Compiler (gccgo or gc). - -gccgoflags 'arg list' - arguments to pass on each gccgo compiler/linker invocation. - -gcflags 'arg list' - arguments to pass on each 5g, 6g, or 8g compiler invocation. - -installsuffix suffix - a suffix to use in the name of the package installation directory, - in order to keep output separate from default builds. - If using the -race flag, the install suffix is automatically set to race - or, if set explicitly, has _race appended to it. - -ldflags 'flag list' - arguments to pass on each 5l, 6l, or 8l linker invocation. - -tags 'tag list' - a list of build tags to consider satisfied during the build. - For more information about build tags, see the description of - build constraints in the documentation for the go/build package. - -The list flags accept a space-separated list of strings. To embed spaces -in an element in the list, surround it with either single or double quotes. - -For more about specifying packages, see 'go help packages'. -For more about where packages and binaries are installed, -run 'go help gopath'. For more about calling between Go and C/C++, -run 'go help c'. - -See also: go install, go get, go clean. - - -Remove object files - -Usage: - - go clean [-i] [-r] [-n] [-x] [build flags] [packages] - -Clean removes object files from package source directories. -The go command builds most objects in a temporary directory, -so go clean is mainly concerned with object files left by other -tools or by manual invocations of go build. - -Specifically, clean removes the following files from each of the -source directories corresponding to the import paths: - - _obj/ old object directory, left from Makefiles - _test/ old test directory, left from Makefiles - _testmain.go old gotest file, left from Makefiles - test.out old test log, left from Makefiles - build.out old test log, left from Makefiles - *.[568ao] object files, left from Makefiles - - DIR(.exe) from go build - DIR.test(.exe) from go test -c - MAINFILE(.exe) from go build MAINFILE.go - *.so from SWIG - -In the list, DIR represents the final path element of the -directory, and MAINFILE is the base name of any Go source -file in the directory that is not included when building -the package. - -The -i flag causes clean to remove the corresponding installed -archive or binary (what 'go install' would create). - -The -n flag causes clean to print the remove commands it would execute, -but not run them. - -The -r flag causes clean to be applied recursively to all the -dependencies of the packages named by the import paths. - -The -x flag causes clean to print remove commands as it executes them. - -For more about build flags, see 'go help build'. - -For more about specifying packages, see 'go help packages'. - - -Print Go environment information - -Usage: - - go env [var ...] - -Env prints Go environment information. - -By default env prints information as a shell script -(on Windows, a batch file). If one or more variable -names is given as arguments, env prints the value of -each named variable on its own line. - - -Run go tool fix on packages - -Usage: - - go fix [packages] - -Fix runs the Go fix command on the packages named by the import paths. - -For more about fix, see 'godoc fix'. -For more about specifying packages, see 'go help packages'. - -To run fix with specific options, run 'go tool fix'. - -See also: go fmt, go vet. - - -Run gofmt on package sources - -Usage: - - go fmt [-n] [-x] [packages] - -Fmt runs the command 'gofmt -l -w' on the packages named -by the import paths. It prints the names of the files that are modified. - -For more about gofmt, see 'godoc gofmt'. -For more about specifying packages, see 'go help packages'. - -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -To run gofmt with specific options, run gofmt itself. - -See also: go fix, go vet. - - -Generate Go files by processing source - -Usage: - - go generate [-run regexp] [file.go... | packages] - -Generate runs commands described by directives within existing -files. Those commands can run any process but the intent is to -create or update Go source files, for instance by running yacc. - -Go generate is never run automatically by go build, go get, go test, -and so on. It must be run explicitly. - -Go generate scans the file for directives, which are lines of -the form, - - //go:generate command argument... - -(note: no leading spaces and no space in "//go") where command -is the generator to be run, corresponding to an executable file -that can be run locally. It must either be in the shell path -(gofmt), a fully qualified path (/usr/you/bin/mytool), or a -command alias, described below. - -Note that go generate does not parse the file, so lines that look -like directives in comments or multiline strings will be treated -as directives. - -The arguments to the directive are space-separated tokens or -double-quoted strings passed to the generator as individual -arguments when it is run. - -Quoted strings use Go syntax and are evaluated before execution; a -quoted string appears as a single argument to the generator. - -Go generate sets several variables when it runs the generator: - - $GOARCH - The execution architecture (arm, amd64, etc.) - $GOOS - The execution operating system (linux, windows, etc.) - $GOFILE - The base name of the file. - $GOPACKAGE - The name of the package of the file containing the directive. - -Other than variable substitution and quoted-string evaluation, no -special processing such as "globbing" is performed on the command -line. - -As a last step before running the command, any invocations of any -environment variables with alphanumeric names, such as $GOFILE or -$HOME, are expanded throughout the command line. The syntax for -variable expansion is $NAME on all operating systems. Due to the -order of evaluation, variables are expanded even inside quoted -strings. If the variable NAME is not set, $NAME expands to the -empty string. - -A directive of the form, - - //go:generate -command xxx args... - -specifies, for the remainder of this source file only, that the -string xxx represents the command identified by the arguments. This -can be used to create aliases or to handle multiword generators. -For example, - - //go:generate -command yacc go tool yacc - -specifies that the command "yacc" represents the generator -"go tool yacc". - -Generate processes packages in the order given on the command line, -one at a time. If the command line lists .go files, they are treated -as a single package. Within a package, generate processes the -source files in a package in file name order, one at a time. Within -a source file, generate runs generators in the order they appear -in the file, one at a time. - -If any generator returns an error exit status, "go generate" skips -all further processing for that package. - -The generator is run in the package's source directory. - -Go generate accepts one specific flag: - - -run="" - TODO: This flag is unimplemented. - if non-empty, specifies a regular expression to - select directives whose command matches the expression. - -It also accepts the standard build flags -v, -n, and -x. -The -v flag prints the names of packages and files as they are -processed. -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -For more about specifying packages, see 'go help packages'. - - -Download and install packages and dependencies - -Usage: - - go get [-d] [-f] [-fix] [-t] [-u] [build flags] [packages] - -Get downloads and installs the packages named by the import paths, -along with their dependencies. - -The -d flag instructs get to stop after downloading the packages; that is, -it instructs get not to install the packages. - -The -f flag, valid only when -u is set, forces get -u not to verify that -each package has been checked out from the source control repository -implied by its import path. This can be useful if the source is a local fork -of the original. - -The -fix flag instructs get to run the fix tool on the downloaded packages -before resolving dependencies or building the code. - -The -t flag instructs get to also download the packages required to build -the tests for the specified packages. - -The -u flag instructs get to use the network to update the named packages -and their dependencies. By default, get uses the network to check out -missing packages but does not use it to look for updates to existing packages. - -Get also accepts build flags to control the installation. See 'go help build'. - -When checking out or updating a package, get looks for a branch or tag -that matches the locally installed version of Go. The most important -rule is that if the local installation is running version "go1", get -searches for a branch or tag named "go1". If no such version exists it -retrieves the most recent version of the package. - -For more about specifying packages, see 'go help packages'. - -For more about how 'go get' finds source code to -download, see 'go help importpath'. - -See also: go build, go install, go clean. - - -Compile and install packages and dependencies - -Usage: - - go install [build flags] [packages] - -Install compiles and installs the packages named by the import paths, -along with their dependencies. - -For more about the build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go get, go clean. - - -List packages - -Usage: - - go list [-e] [-f format] [-json] [build flags] [packages] - -List lists the packages named by the import paths, one per line. - -The default output shows the package import path: - - code.google.com/p/google-api-go-client/books/v1 - code.google.com/p/goauth2/oauth - code.google.com/p/sqlite - -The -f flag specifies an alternate format for the list, using the -syntax of package template. The default output is equivalent to -f -'{{.ImportPath}}'. The struct being passed to the template is: - - type Package struct { - Dir string // directory containing package sources - ImportPath string // import path of package in dir - ImportComment string // path in import comment on package statement - Name string // package name - Doc string // package documentation string - Target string // install path - Goroot bool // is this package in the Go root? - Standard bool // is this package part of the standard Go library? - Stale bool // would 'go install' do anything for this package? - Root string // Go root or Go path dir containing this package - - // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go sources files that import "C" - IgnoredGoFiles []string // .go sources ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - - // Cgo directives - CgoCFLAGS []string // cgo: flags for C compiler - CgoCPPFLAGS []string // cgo: flags for C preprocessor - CgoCXXFLAGS []string // cgo: flags for C++ compiler - CgoLDFLAGS []string // cgo: flags for linker - CgoPkgConfig []string // cgo: pkg-config names - - // Dependency information - Imports []string // import paths used by this package - Deps []string // all (recursively) imported dependencies - - // Error information - Incomplete bool // this package or a dependency has an error - Error *PackageError // error loading package - DepsErrors []*PackageError // errors loading dependencies - - TestGoFiles []string // _test.go files in package - TestImports []string // imports from TestGoFiles - XTestGoFiles []string // _test.go files outside package - XTestImports []string // imports from XTestGoFiles - } - -The template function "join" calls strings.Join. - -The template function "context" returns the build context, defined as: - - type Context struct { - GOARCH string // target architecture - GOOS string // target operating system - GOROOT string // Go root - GOPATH string // Go path - CgoEnabled bool // whether cgo can be used - UseAllFiles bool // use files regardless of +build lines, file names - Compiler string // compiler to assume when computing target paths - BuildTags []string // build constraints to match in +build lines - ReleaseTags []string // releases the current release is compatible with - InstallSuffix string // suffix to use in the name of the install dir - } - -For more information about the meaning of these fields see the documentation -for the go/build package's Context type. - -The -json flag causes the package data to be printed in JSON format -instead of using the template format. - -The -e flag changes the handling of erroneous packages, those that -cannot be found or are malformed. By default, the list command -prints an error to standard error for each erroneous package and -omits the packages from consideration during the usual printing. -With the -e flag, the list command never prints errors to standard -error and instead processes the erroneous packages with the usual -printing. Erroneous packages will have a non-empty ImportPath and -a non-nil Error field; other information may or may not be missing -(zeroed). - -For more about build flags, see 'go help build'. - -For more about specifying packages, see 'go help packages'. - - -Compile and run Go program - -Usage: - - go run [build flags] [-exec xprog] gofiles... [arguments...] - -Run compiles and runs the main package comprising the named Go source files. -A Go source file is defined to be a file ending in a literal ".go" suffix. - -By default, 'go run' runs the compiled binary directly: 'a.out arguments...'. -If the -exec flag is given, 'go run' invokes the binary using xprog: 'xprog a.out arguments...'. -If the -exec flag is not given, GOOS or GOARCH is different from the system -default, and a program named go_$GOOS_$GOARCH_exec can be found -on the current search path, 'go run' invokes the binary using that program, -for example 'go_nacl_386_exec a.out arguments...'. This allows execution of -cross-compiled programs when a simulator or other execution method is -available. - -For more about build flags, see 'go help build'. - -See also: go build. - - -Test packages - -Usage: - - go test [-c] [-i] [build and test flags] [packages] [flags for test binary] - -'Go test' automates testing the packages named by the import paths. -It prints a summary of the test results in the format: - - ok archive/tar 0.011s - FAIL archive/zip 0.022s - ok compress/gzip 0.033s - ... - -followed by detailed output for each failed package. - -'Go test' recompiles each package along with any files with names matching -the file pattern "*_test.go". -Files whose names begin with "_" (including "_test.go") or "." are ignored. -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. -Each listed package causes the execution of a separate test binary. - -Test files that declare a package with the suffix "_test" will be compiled as a -separate package, and then linked and run with the main test binary. - -By default, go test needs no arguments. It compiles and tests the package -with source in the current directory, including tests, and runs the tests. - -The package is built in a temporary directory so it does not interfere with the -non-test installation. - -In addition to the build flags, the flags handled by 'go test' itself are: +package main +var cmdDoc = &Command{ + Run: runDoc, + UsageLine: "doc [-u] [-c] [package|[package.]symbol[.method]]", + CustomFlags: true, + Short: "show documentation for package or symbol", + Long: ` +Doc prints the documentation comments associated with the item identified by its +arguments (a package, const, func, type, var, or method) followed by a one-line +summary of each of the first-level items "under" that item (package-level +declarations for a package, methods for a type, etc.). + +Doc accepts zero, one, or two arguments. + +Given no arguments, that is, when run as + + go doc + +it prints the package documentation for the package in the current directory. +If the package is a command (package main), the exported symbols of the package +are elided from the presentation unless the -cmd flag is provided. + +When run with one argument, the argument is treated as a Go-syntax-like +representation of the item to be documented. What the argument selects depends +on what is installed in GOROOT and GOPATH, as well as the form of the argument, +which is schematically one of these: + + go doc <pkg> + go doc <sym>[.<method>] + go doc [<pkg>].<sym>[.<method>] + +The first item in this list matched by the argument is the one whose +documentation is printed. (See the examples below.) For packages, the order of +scanning is determined lexically, but the GOROOT tree is always scanned before +GOPATH. + +If there is no package specified or matched, the package in the current +directory is selected, so "go doc Foo" shows the documentation for symbol Foo in +the current package. + +The package path must be either a qualified path or a proper suffix of a +path. The go tool's usual package mechanism does not apply: package path +elements like . and ... are not implemented by go doc. + +When run with two arguments, the first must be a full package path (not just a +suffix), and the second is a symbol or symbol and method; this is similar to the +syntax accepted by godoc: + + go doc <pkg> <sym>[.<method>] + +In all forms, when matching symbols, lower-case letters in the argument match +either case but upper-case letters match exactly. This means that there may be +multiple matches of a lower-case argument in a package if different symbols have +different cases. If this occurs, documentation for all matches is printed. + +Examples: + go doc + Show documentation for current package. + go doc Foo + Show documentation for Foo in the current package. + (Foo starts with a capital letter so it cannot match + a package path.) + go doc encoding/json + Show documentation for the encoding/json package. + go doc json + Shorthand for encoding/json. + go doc json.Number (or go doc json.number) + Show documentation and method summary for json.Number. + go doc json.Number.Int64 (or go doc json.number.int64) + Show documentation for json.Number's Int64 method. + go doc cmd/doc + Show package docs for the doc command. + go doc -cmd cmd/doc + Show package docs and exported symbols within the doc command. + go doc template.new + Show documentation for html/template's New function. + (html/template is lexically before text/template) + go doc text/template.new # One argument + Show documentation for text/template's New function. + go doc text/template new # Two arguments + Show documentation for text/template's New function. + +Flags: -c - Compile the test binary to pkg.test but do not run it - (where pkg is the last element of the package's import path). - The file name can be changed with the -o flag. - - -exec xprog - Run the test binary using xprog. The behavior is the same as - in 'go run'. See 'go help run' for details. - - -i - Install packages that are dependencies of the test. - Do not run the test. - - -o file - Compile the test binary to the named file. - The test still runs (unless -c or -i is specified). - - -The test binary also accepts flags that control execution of the test; these -flags are also accessible by 'go test'. See 'go help testflag' for details. - -If the test binary needs any other flags, they should be presented after the -package names. The go tool treats as a flag the first argument that begins with -a minus sign that it does not recognize itself; that argument and all subsequent -arguments are passed as arguments to the test binary. - -For more about build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go vet. - - -Run specified go tool - -Usage: - - go tool [-n] command [args...] - -Tool runs the go tool command identified by the arguments. -With no arguments it prints the list of known tools. - -The -n flag causes tool to print the command that would be -executed but not execute it. - -For more about each tool command, see 'go tool command -h'. - - -Print Go version - -Usage: - - go version - -Version prints the Go version, as reported by runtime.Version. - - -Run go tool vet on packages - -Usage: - - go vet [-n] [-x] [packages] - -Vet runs the Go vet command on the packages named by the import paths. - -For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. -For more about specifying packages, see 'go help packages'. - -To run the vet tool with specific options, run 'go tool vet'. - -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -See also: go fmt, go fix. - - -Calling between Go and C - -There are two different ways to call between Go and C/C++ code. - -The first is the cgo tool, which is part of the Go distribution. For -information on how to use it see the cgo documentation (godoc cmd/cgo). - -The second is the SWIG program, which is a general tool for -interfacing between languages. For information on SWIG see -http://swig.org/. When running go build, any file with a .swig -extension will be passed to SWIG. Any file with a .swigcxx extension -will be passed to SWIG with the -c++ option. - -When either cgo or SWIG is used, go build will pass any .c, .m, .s, -or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++ -compiler. The CC or CXX environment variables may be set to determine -the C or C++ compiler, respectively, to use. - - -File types - -The go command examines the contents of a restricted set of files -in each directory. It identifies which files to examine based on -the extension of the file name. These extensions are: - - .go - Go source files. - .c, .h - C source files. - If the package uses cgo, these will be compiled with the - OS-native compiler (typically gcc); otherwise they will be - compiled with the Go-specific support compiler, - 5c, 6c, or 8c, etc. as appropriate. - .cc, .cpp, .cxx, .hh, .hpp, .hxx - C++ source files. Only useful with cgo or SWIG, and always - compiled with the OS-native compiler. - .m - Objective-C source files. Only useful with cgo, and always - compiled with the OS-native compiler. - .s, .S - Assembler source files. - If the package uses cgo, these will be assembled with the - OS-native assembler (typically gcc (sic)); otherwise they - will be assembled with the Go-specific support assembler, - 5a, 6a, or 8a, etc., as appropriate. - .swig, .swigcxx - SWIG definition files. - .syso - System object files. - -Files of each of these types except .syso may contain build -constraints, but the go command stops scanning for build constraints -at the first item in the file that is not a blank line or //-style -line comment. - - -GOPATH environment variable - -The Go path is used to resolve import statements. -It is implemented by and documented in the go/build package. - -The GOPATH environment variable lists places to look for Go code. -On Unix, the value is a colon-separated string. -On Windows, the value is a semicolon-separated string. -On Plan 9, the value is a list. - -GOPATH must be set to get, build and install packages outside the -standard Go tree. - -Each directory listed in GOPATH must have a prescribed structure: - -The src/ directory holds source code. The path below 'src' -determines the import path or executable name. - -The pkg/ directory holds installed package objects. -As in the Go tree, each target operating system and -architecture pair has its own subdirectory of pkg -(pkg/GOOS_GOARCH). - -If DIR is a directory listed in the GOPATH, a package with -source in DIR/src/foo/bar can be imported as "foo/bar" and -has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". - -The bin/ directory holds compiled commands. -Each command is named for its source directory, but only -the final element, not the entire path. That is, the -command with source in DIR/src/foo/quux is installed into -DIR/bin/quux, not DIR/bin/foo/quux. The foo/ is stripped -so that you can add DIR/bin to your PATH to get at the -installed commands. If the GOBIN environment variable is -set, commands are installed to the directory it names instead -of DIR/bin. - -Here's an example directory layout: - - GOPATH=/home/user/gocode - - /home/user/gocode/ - src/ - foo/ - bar/ (go code in package bar) - x.go - quux/ (go code in package main) - y.go - bin/ - quux (installed command) - pkg/ - linux_amd64/ - foo/ - bar.a (installed package object) - -Go searches each directory listed in GOPATH to find source code, -but new packages are always downloaded into the first directory -in the list. - - -Import path syntax - -An import path (see 'go help packages') denotes a package -stored in the local file system. In general, an import path denotes -either a standard package (such as "unicode/utf8") or a package -found in one of the work spaces (see 'go help gopath'). - -Relative import paths - -An import path beginning with ./ or ../ is called a relative path. -The toolchain supports relative import paths as a shortcut in two ways. - -First, a relative path can be used as a shorthand on the command line. -If you are working in the directory containing the code imported as -"unicode" and want to run the tests for "unicode/utf8", you can type -"go test ./utf8" instead of needing to specify the full path. -Similarly, in the reverse situation, "go test .." will test "unicode" from -the "unicode/utf8" directory. Relative patterns are also allowed, like -"go test ./..." to test all subdirectories. See 'go help packages' for details -on the pattern syntax. - -Second, if you are compiling a Go program not in a work space, -you can use a relative path in an import statement in that program -to refer to nearby code also not in a work space. -This makes it easy to experiment with small multipackage programs -outside of the usual work spaces, but such programs cannot be -installed with "go install" (there is no work space in which to install them), -so they are rebuilt from scratch each time they are built. -To avoid ambiguity, Go programs cannot use relative import paths -within a work space. - -Remote import paths - -Certain import paths also -describe how to obtain the source code for the package using -a revision control system. - -A few common code hosting sites have special syntax: - - Bitbucket (Git, Mercurial) - - import "bitbucket.org/user/project" - import "bitbucket.org/user/project/sub/directory" - - GitHub (Git) - - import "github.com/user/project" - import "github.com/user/project/sub/directory" - - Google Code Project Hosting (Git, Mercurial, Subversion) - - import "code.google.com/p/project" - import "code.google.com/p/project/sub/directory" - - import "code.google.com/p/project.subrepository" - import "code.google.com/p/project.subrepository/sub/directory" - - Launchpad (Bazaar) - - import "launchpad.net/project" - import "launchpad.net/project/series" - import "launchpad.net/project/series/sub/directory" - - import "launchpad.net/~user/project/branch" - import "launchpad.net/~user/project/branch/sub/directory" - - IBM DevOps Services (Git) - - import "hub.jazz.net/git/user/project" - import "hub.jazz.net/git/user/project/sub/directory" - -For code hosted on other servers, import paths may either be qualified -with the version control type, or the go tool can dynamically fetch -the import path over https/http and discover where the code resides -from a <meta> tag in the HTML. - -To declare the code location, an import path of the form - - repository.vcs/path - -specifies the given repository, with or without the .vcs suffix, -using the named version control system, and then the path inside -that repository. The supported version control systems are: - - Bazaar .bzr - Git .git - Mercurial .hg - Subversion .svn - -For example, - - import "example.org/user/foo.hg" - -denotes the root directory of the Mercurial repository at -example.org/user/foo or foo.hg, and - - import "example.org/repo.git/foo/bar" - -denotes the foo/bar directory of the Git repository at -example.org/repo or repo.git. - -When a version control system supports multiple protocols, -each is tried in turn when downloading. For example, a Git -download tries git://, then https://, then http://. - -If the import path is not a known code hosting site and also lacks a -version control qualifier, the go tool attempts to fetch the import -over https/http and looks for a <meta> tag in the document's HTML -<head>. - -The meta tag has the form: - - <meta name="go-import" content="import-prefix vcs repo-root"> - -The import-prefix is the import path corresponding to the repository -root. It must be a prefix or an exact match of the package being -fetched with "go get". If it's not an exact match, another http -request is made at the prefix to verify the <meta> tags match. - -The vcs is one of "git", "hg", "svn", etc, - -The repo-root is the root of the version control system -containing a scheme and not containing a .vcs qualifier. - -For example, - - import "example.org/pkg/foo" - -will result in the following request(s): - - https://example.org/pkg/foo?go-get=1 (preferred) - http://example.org/pkg/foo?go-get=1 (fallback) - -If that page contains the meta tag - - <meta name="go-import" content="example.org git https://code.org/r/p/exproj"> - -the go tool will verify that https://example.org/?go-get=1 contains the -same meta tag and then git clone https://code.org/r/p/exproj into -GOPATH/src/example.org. - -New downloaded packages are written to the first directory -listed in the GOPATH environment variable (see 'go help gopath'). - -The go command attempts to download the version of the -package appropriate for the Go release being used. -Run 'go help get' for more. - -Import path checking - -When the custom import path feature described above redirects to a -known code hosting site, each of the resulting packages has two possible -import paths, using the custom domain or the known hosting site. - -A package statement is said to have an "import comment" if it is immediately -followed (before the next newline) by a comment of one of these two forms: - - package math // import "path" - package math /* import "path" * / - -The go command will refuse to install a package with an import comment -unless it is being referred to by that import path. In this way, import comments -let package authors make sure the custom import path is used and not a -direct path to the underlying code hosting site. - -See https://golang.org/s/go14customimport for details. - - -Description of package lists - -Many commands apply to a set of packages: - - go action [packages] - -Usually, [packages] is a list of import paths. - -An import path that is a rooted path or that begins with -a . or .. element is interpreted as a file system path and -denotes the package in that directory. - -Otherwise, the import path P denotes the package found in -the directory DIR/src/P for some DIR listed in the GOPATH -environment variable (see 'go help gopath'). - -If no import paths are given, the action applies to the -package in the current directory. - -There are three reserved names for paths that should not be used -for packages to be built with the go tool: - -- "main" denotes the top-level package in a stand-alone executable. - -- "all" expands to all package directories found in all the GOPATH -trees. For example, 'go list all' lists all the packages on the local -system. - -- "std" is like all but expands to just the packages in the standard -Go library. - -An import path is a pattern if it includes one or more "..." wildcards, -each of which can match any string, including the empty string and -strings containing slashes. Such a pattern expands to all package -directories found in the GOPATH trees with names matching the -patterns. As a special case, x/... matches x as well as x's subdirectories. -For example, net/... expands to net and packages in its subdirectories. - -An import path can also name a package to be downloaded from -a remote repository. Run 'go help importpath' for details. - -Every package in a program must have a unique import path. -By convention, this is arranged by starting each path with a -unique prefix that belongs to you. For example, paths used -internally at Google all begin with 'google', and paths -denoting remote repositories begin with the path to the code, -such as 'code.google.com/p/project'. - -As a special case, if the package list is a list of .go files from a -single directory, the command is applied to a single synthesized -package made up of exactly those files, ignoring any build constraints -in those files and ignoring any other files in the directory. - -Directory and file names that begin with "." or "_" are ignored -by the go tool, as are directories named "testdata". - - -Description of testing flags - -The 'go test' command takes both flags that apply to 'go test' itself -and flags that apply to the resulting test binary. - -Several of the flags control profiling and write an execution profile -suitable for "go tool pprof"; run "go tool pprof help" for more -information. The --alloc_space, --alloc_objects, and --show_bytes -options of pprof control how the information is presented. - -The following flags are recognized by the 'go test' command and -control the execution of any test: - - -bench regexp - Run benchmarks matching the regular expression. - By default, no benchmarks run. To run all benchmarks, - use '-bench .' or '-bench=.'. - - -benchmem - Print memory allocation statistics for benchmarks. - - -benchtime t - Run enough iterations of each benchmark to take t, specified - as a time.Duration (for example, -benchtime 1h30s). - The default is 1 second (1s). - - -blockprofile block.out - Write a goroutine blocking profile to the specified file - when all tests are complete. - Writes test binary as -c would. - - -blockprofilerate n - Control the detail provided in goroutine blocking profiles by - calling runtime.SetBlockProfileRate with n. - See 'godoc runtime SetBlockProfileRate'. - The profiler aims to sample, on average, one blocking event every - n nanoseconds the program spends blocked. By default, - if -test.blockprofile is set without this flag, all blocking events - are recorded, equivalent to -test.blockprofilerate=1. - - -cover - Enable coverage analysis. - - -covermode set,count,atomic - Set the mode for coverage analysis for the package[s] - being tested. The default is "set" unless -race is enabled, - in which case it is "atomic". - The values: - set: bool: does this statement run? - count: int: how many times does this statement run? - atomic: int: count, but correct in multithreaded tests; - significantly more expensive. - Sets -cover. - - -coverpkg pkg1,pkg2,pkg3 - Apply coverage analysis in each test to the given list of packages. - The default is for each test to analyze only the package being tested. - Packages are specified as import paths. - Sets -cover. - - -coverprofile cover.out - Write a coverage profile to the file after all tests have passed. - Sets -cover. - - -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value - of GOMAXPROCS. - - -cpuprofile cpu.out - Write a CPU profile to the specified file before exiting. - Writes test binary as -c would. - - -memprofile mem.out - Write a memory profile to the file after all tests have passed. - Writes test binary as -c would. - - -memprofilerate n - Enable more precise (and expensive) memory profiles by setting - runtime.MemProfileRate. See 'godoc runtime MemProfileRate'. - To profile all memory allocations, use -test.memprofilerate=1 - and pass --alloc_space flag to the pprof tool. - - -outputdir directory - Place output files from profiling in the specified directory, - by default the directory in which "go test" is running. - - -parallel n - Allow parallel execution of test functions that call t.Parallel. - The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. - - -run regexp - Run only those tests and examples matching the regular - expression. - - -short - Tell long-running tests to shorten their run time. - It is off by default but set during all.bash so that installing - the Go tree can run a sanity check but not spend time running - exhaustive tests. - - -timeout t - If a test runs longer than t, panic. - - -v - Verbose output: log all tests as they are run. Also print all - text from Log and Logf calls even if the test succeeds. - -The test binary, called pkg.test where pkg is the name of the -directory containing the package sources, can be invoked directly -after building it with 'go test -c'. When invoking the test binary -directly, each of the standard flag names must be prefixed with 'test.', -as in -test.run=TestMyFunc or -test.v. - -When running 'go test', flags not listed above are passed through -unaltered. For instance, the command - - go test -x -v -cpuprofile=prof.out -dir=testdata -update - -will compile the test binary and then run it as - - pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update - -The test flags that generate profiles (other than for coverage) also -leave the test binary in pkg.test for use when analyzing the profiles. - -Flags not recognized by 'go test' must be placed after any specified packages. - - -Description of testing functions - -The 'go test' command expects to find test, benchmark, and example functions -in the "*_test.go" files corresponding to the package under test. - -A test function is one named TestXXX (where XXX is any alphanumeric string -not starting with a lower case letter) and should have the signature, - - func TestXXX(t *testing.T) { ... } - -A benchmark function is one named BenchmarkXXX and should have the signature, - - func BenchmarkXXX(b *testing.B) { ... } - -An example function is similar to a test function but, instead of using -*testing.T to report success or failure, prints output to os.Stdout. -That output is compared against the function's "Output:" comment, which -must be the last comment in the function body (see example below). An -example with no such comment, or with no text after "Output:" is compiled -but not executed. - -Godoc displays the body of ExampleXXX to demonstrate the use -of the function, constant, or variable XXX. An example of a method M with -receiver type T or *T is named ExampleT_M. There may be multiple examples -for a given function, constant, or variable, distinguished by a trailing _xxx, -where xxx is a suffix not beginning with an upper case letter. - -Here is an example of an example: - - func ExamplePrintln() { - Println("The output of\nthis example.") - // Output: The output of - // this example. - } - -The entire test file is presented as the example when it contains a single -example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. - -See the documentation of the testing package for more information. - - -*/ -package main + Respect case when matching symbols. + -cmd + Treat a command (package main) like a regular package. + Otherwise package main's exported symbols are hidden + when showing the package's top-level documentation. + -u + Show documentation for unexported as well as exported + symbols and methods. +`, +} + +func runDoc(cmd *Command, args []string) { + run(buildToolExec, tool("doc"), args) +} diff --git a/libgo/go/cmd/go/env.go b/libgo/go/cmd/go/env.go index 26d37df4f9b..600accac03f 100644 --- a/libgo/go/cmd/go/env.go +++ b/libgo/go/cmd/go/env.go @@ -36,7 +36,6 @@ func mkEnv() []envVar { env := []envVar{ {"GOARCH", goarch}, {"GOBIN", gobin}, - {"GOCHAR", archChar}, {"GOEXE", exeSuffix}, {"GOHOSTARCH", runtime.GOARCH}, {"GOHOSTOS", runtime.GOOS}, @@ -45,6 +44,7 @@ func mkEnv() []envVar { {"GORACE", os.Getenv("GORACE")}, {"GOROOT", goroot}, {"GOTOOLDIR", toolDir}, + {"GO15VENDOREXPERIMENT", os.Getenv("GO15VENDOREXPERIMENT")}, // disable escape codes in clang errors {"TERM", "dumb"}, diff --git a/libgo/go/cmd/go/fix.go b/libgo/go/cmd/go/fix.go index 8736cce3e2a..94fd22e3c21 100644 --- a/libgo/go/cmd/go/fix.go +++ b/libgo/go/cmd/go/fix.go @@ -11,7 +11,7 @@ var cmdFix = &Command{ Long: ` Fix runs the Go fix command on the packages named by the import paths. -For more about fix, see 'godoc fix'. +For more about fix, see 'go doc cmd/fix'. For more about specifying packages, see 'go help packages'. To run fix with specific options, run 'go tool fix'. @@ -25,6 +25,6 @@ func runFix(cmd *Command, args []string) { // Use pkg.gofiles instead of pkg.Dir so that // the command only applies to this package, // not to packages in subdirectories. - run(stringList(tool("fix"), relPaths(pkg.allgofiles))) + run(stringList(buildToolExec, tool("fix"), relPaths(pkg.allgofiles))) } } diff --git a/libgo/go/cmd/go/fmt.go b/libgo/go/cmd/go/fmt.go index 65dc3ca5990..57c02ad2647 100644 --- a/libgo/go/cmd/go/fmt.go +++ b/libgo/go/cmd/go/fmt.go @@ -4,6 +4,11 @@ package main +import ( + "os" + "path/filepath" +) + func init() { addBuildFlagsNX(cmdFmt) } @@ -16,7 +21,7 @@ var cmdFmt = &Command{ Fmt runs the command 'gofmt -l -w' on the packages named by the import paths. It prints the names of the files that are modified. -For more about gofmt, see 'godoc gofmt'. +For more about gofmt, see 'go doc cmd/gofmt'. For more about specifying packages, see 'go help packages'. The -n flag prints commands that would be executed. @@ -29,10 +34,31 @@ See also: go fix, go vet. } func runFmt(cmd *Command, args []string) { + gofmt := gofmtPath() for _, pkg := range packages(args) { // Use pkg.gofiles instead of pkg.Dir so that // the command only applies to this package, // not to packages in subdirectories. - run(stringList("gofmt", "-l", "-w", relPaths(pkg.allgofiles))) + run(stringList(gofmt, "-l", "-w", relPaths(pkg.allgofiles))) } } + +func gofmtPath() string { + gofmt := "gofmt" + if toolIsWindows { + gofmt += toolWindowsExtension + } + + gofmtPath := filepath.Join(gobin, gofmt) + if _, err := os.Stat(gofmtPath); err == nil { + return gofmtPath + } + + gofmtPath = filepath.Join(goroot, "bin", gofmt) + if _, err := os.Stat(gofmtPath); err == nil { + return gofmtPath + } + + // fallback to looking for gofmt in $PATH + return "gofmt" +} diff --git a/libgo/go/cmd/go/generate.go b/libgo/go/cmd/go/generate.go index 3c0af8760b5..efdc229b228 100644 --- a/libgo/go/cmd/go/generate.go +++ b/libgo/go/cmd/go/generate.go @@ -13,11 +13,11 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strconv" "strings" "unicode" - "unicode/utf8" ) var cmdGenerate = &Command{ @@ -62,8 +62,12 @@ Go generate sets several variables when it runs the generator: The execution operating system (linux, windows, etc.) $GOFILE The base name of the file. + $GOLINE + The line number of the directive in the source file. $GOPACKAGE The name of the package of the file containing the directive. + $DOLLAR + A dollar sign. Other than variable substitution and quoted-string evaluation, no special processing such as "globbing" is performed on the command @@ -106,9 +110,10 @@ The generator is run in the package's source directory. Go generate accepts one specific flag: -run="" - TODO: This flag is unimplemented. - if non-empty, specifies a regular expression to - select directives whose command matches the expression. + if non-empty, specifies a regular expression to select + directives whose full original source text (excluding + any trailing spaces and final newline) matches the + expression. It also accepts the standard build flags -v, -n, and -x. The -v flag prints the names of packages and files as they are @@ -120,7 +125,10 @@ For more about specifying packages, see 'go help packages'. `, } -var generateRunFlag string // generate -run flag +var ( + generateRunFlag string // generate -run flag + generateRunRE *regexp.Regexp // compiled expression for -run +) func init() { addBuildFlags(cmdGenerate) @@ -128,6 +136,13 @@ func init() { } func runGenerate(cmd *Command, args []string) { + if generateRunFlag != "" { + var err error + generateRunRE, err = regexp.Compile(generateRunFlag) + if err != nil { + log.Fatalf("generate: %s", err) + } + } // Even if the arguments are .go files, this loop suffices. for _, pkg := range packages(args) { for _, file := range pkg.gofiles { @@ -163,7 +178,7 @@ type Generator struct { file string // base name of file. pkg string commands map[string][]string - lineNum int + lineNum int // current line number. } // run runs the generators in the current file. @@ -221,6 +236,11 @@ func (g *Generator) run() (ok bool) { if !isGoGenerate(buf) { continue } + if generateRunFlag != "" { + if !generateRunRE.Match(bytes.TrimSpace(buf)) { + continue + } + } words := g.split(string(buf)) if len(words) == 0 { @@ -306,7 +326,7 @@ Words: } // Substitute environment variables. for i, word := range words { - words[i] = g.expandEnv(word) + words[i] = os.Expand(word, g.expandVar) } return words } @@ -322,38 +342,25 @@ func (g *Generator) errorf(format string, args ...interface{}) { panic(stop) } -// expandEnv expands any $XXX invocations in word. -func (g *Generator) expandEnv(word string) string { - if !strings.ContainsRune(word, '$') { - return word - } - var buf bytes.Buffer - var w int - var r rune - for i := 0; i < len(word); i += w { - r, w = utf8.DecodeRuneInString(word[i:]) - if r != '$' { - buf.WriteRune(r) - continue - } - w += g.identLength(word[i+w:]) - envVar := word[i+1 : i+w] - var sub string - switch envVar { - case "GOARCH": - sub = runtime.GOARCH - case "GOOS": - sub = runtime.GOOS - case "GOFILE": - sub = g.file - case "GOPACKAGE": - sub = g.pkg - default: - sub = os.Getenv(envVar) - } - buf.WriteString(sub) +// expandVar expands the $XXX invocation in word. It is called +// by os.Expand. +func (g *Generator) expandVar(word string) string { + switch word { + case "GOARCH": + return buildContext.GOARCH + case "GOOS": + return buildContext.GOOS + case "GOFILE": + return g.file + case "GOLINE": + return fmt.Sprint(g.lineNum) + case "GOPACKAGE": + return g.pkg + case "DOLLAR": + return "$" + default: + return os.Getenv(word) } - return buf.String() } // identLength returns the length of the identifier beginning the string. @@ -395,7 +402,7 @@ func (g *Generator) exec(words []string) { "GOFILE=" + g.file, "GOPACKAGE=" + g.pkg, } - cmd.Env = mergeEnvLists(env, os.Environ()) + cmd.Env = mergeEnvLists(env, origEnv) err := cmd.Run() if err != nil { g.errorf("running %q: %s", words[0], err) diff --git a/libgo/go/cmd/go/generate_test.go b/libgo/go/cmd/go/generate_test.go index 2ec548630ac..169d71ca812 100644 --- a/libgo/go/cmd/go/generate_test.go +++ b/libgo/go/cmd/go/generate_test.go @@ -26,6 +26,7 @@ var splitTests = []splitTest{ {"$GOPACKAGE", []string{"sys"}}, {"a $XXNOTDEFINEDXX b", []string{"a", "", "b"}}, {"/$XXNOTDEFINED/", []string{"//"}}, + {"/$DOLLAR/", []string{"/$/"}}, {"yacc -o $GOARCH/yacc_$GOFILE", []string{"go", "tool", "yacc", "-o", runtime.GOARCH + "/yacc_proc.go"}}, } diff --git a/libgo/go/cmd/go/get.go b/libgo/go/cmd/go/get.go index 50e0ca93bf4..e95201a6930 100644 --- a/libgo/go/cmd/go/get.go +++ b/libgo/go/cmd/go/get.go @@ -16,7 +16,7 @@ import ( ) var cmdGet = &Command{ - UsageLine: "get [-d] [-f] [-fix] [-t] [-u] [build flags] [packages]", + UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]", Short: "download and install packages and dependencies", Long: ` Get downloads and installs the packages named by the import paths, @@ -33,6 +33,9 @@ of the original. The -fix flag instructs get to run the fix tool on the downloaded packages before resolving dependencies or building the code. +The -insecure flag permits fetching from repositories and resolving +custom domains using insecure schemes such as HTTP. Use with caution. + The -t flag instructs get to also download the packages required to build the tests for the specified packages. @@ -48,6 +51,10 @@ rule is that if the local installation is running version "go1", get searches for a branch or tag named "go1". If no such version exists it retrieves the most recent version of the package. +If the vendoring experiment is enabled (see 'go help gopath'), +then when go get checks out or updates a Git repository, +it also updates any git submodules referenced by the repository. + For more about specifying packages, see 'go help packages'. For more about how 'go get' finds source code to @@ -62,6 +69,7 @@ var getF = cmdGet.Flag.Bool("f", false, "") var getT = cmdGet.Flag.Bool("t", false, "") var getU = cmdGet.Flag.Bool("u", false, "") var getFix = cmdGet.Flag.Bool("fix", false, "") +var getInsecure = cmdGet.Flag.Bool("insecure", false, "") func init() { addBuildFlags(cmdGet) @@ -73,10 +81,20 @@ func runGet(cmd *Command, args []string) { fatalf("go get: cannot use -f flag without -u") } + // Disable any prompting for passwords by Git. + // Only has an effect for 2.3.0 or later, but avoiding + // the prompt in earlier versions is just too hard. + // See golang.org/issue/9341. + os.Setenv("GIT_TERMINAL_PROMPT", "0") + // Phase 1. Download/update. var stk importStack + mode := 0 + if *getT { + mode |= getTestDeps + } for _, arg := range downloadPaths(args) { - download(arg, &stk, *getT) + download(arg, nil, &stk, mode) } exitIfErrors() @@ -92,12 +110,13 @@ func runGet(cmd *Command, args []string) { } args = importPaths(args) + packagesForBuild(args) // Phase 3. Install. if *getD { // Download only. // Check delayed until now so that importPaths - // has a chance to print errors. + // and packagesForBuild have a chance to print errors. return } @@ -148,13 +167,30 @@ var downloadRootCache = map[string]bool{} // download runs the download half of the get command // for the package named by the argument. -func download(arg string, stk *importStack, getTestDeps bool) { - p := loadPackage(arg, stk) +func download(arg string, parent *Package, stk *importStack, mode int) { + load := func(path string, mode int) *Package { + if parent == nil { + return loadPackage(path, stk) + } + return loadImport(path, parent.Dir, parent, stk, nil, mode) + } + + p := load(arg, mode) if p.Error != nil && p.Error.hard { errorf("%s", p.Error) return } + // loadPackage inferred the canonical ImportPath from arg. + // Use that in the following to prevent hysteresis effects + // in e.g. downloadCache and packageCache. + // This allows invocations such as: + // mkdir -p $GOPATH/src/github.com/user + // cd $GOPATH/src/github.com/user + // go get ./foo + // see: golang.org/issue/9767 + arg = p.ImportPath + // There's nothing to do if this is a package in the standard library. if p.Standard { return @@ -163,7 +199,7 @@ func download(arg string, stk *importStack, getTestDeps bool) { // Only process each package once. // (Unless we're fetching test dependencies for this package, // in which case we want to process it again.) - if downloadCache[arg] && !getTestDeps { + if downloadCache[arg] && mode&getTestDeps == 0 { return } downloadCache[arg] = true @@ -175,7 +211,7 @@ func download(arg string, stk *importStack, getTestDeps bool) { // Download if the package is missing, or update if we're using -u. if p.Dir == "" || *getU { // The actual download. - stk.push(p.ImportPath) + stk.push(arg) err := downloadPackage(p) if err != nil { errorf("%s", &PackageError{ImportStack: stk.copy(), Err: err.Error()}) @@ -183,6 +219,17 @@ func download(arg string, stk *importStack, getTestDeps bool) { return } + // Warn that code.google.com is shutting down. We + // issue the warning here because this is where we + // have the import stack. + if strings.HasPrefix(p.ImportPath, "code.google.com") { + fmt.Fprintf(os.Stderr, "warning: code.google.com is shutting down; import path %v will stop working\n", p.ImportPath) + if len(*stk) > 1 { + fmt.Fprintf(os.Stderr, "warning: package %v\n", strings.Join(*stk, "\n\timports ")) + } + } + stk.pop() + args := []string{arg} // If the argument has a wildcard in it, re-evaluate the wildcard. // We delay this until after reloadPackage so that the old entry @@ -208,9 +255,10 @@ func download(arg string, stk *importStack, getTestDeps bool) { pkgs = pkgs[:0] for _, arg := range args { - stk.push(arg) - p := loadPackage(arg, stk) - stk.pop() + // Note: load calls loadPackage or loadImport, + // which push arg onto stk already. + // Do not push here too, or else stk will say arg imports arg. + p := load(arg, mode) if p.Error != nil { errorf("%s", p.Error) continue @@ -223,7 +271,7 @@ func download(arg string, stk *importStack, getTestDeps bool) { // due to wildcard expansion. for _, p := range pkgs { if *getFix { - run(stringList(tool("fix"), relPaths(p.allgofiles))) + run(buildToolExec, stringList(tool("fix"), relPaths(p.allgofiles))) // The imports might have changed, so reload again. p = reloadPackage(arg, stk) @@ -240,18 +288,31 @@ func download(arg string, stk *importStack, getTestDeps bool) { } // Process dependencies, now that we know what they are. - for _, dep := range p.deps { + for _, path := range p.Imports { + if path == "C" { + continue + } // Don't get test dependencies recursively. - download(dep.ImportPath, stk, false) + // Imports is already vendor-expanded. + download(path, p, stk, 0) } - if getTestDeps { + if mode&getTestDeps != 0 { // Process test dependencies when -t is specified. // (Don't get test dependencies for test dependencies.) + // We pass useVendor here because p.load does not + // vendor-expand TestImports and XTestImports. + // The call to loadImport inside download needs to do that. for _, path := range p.TestImports { - download(path, stk, false) + if path == "C" { + continue + } + download(path, p, stk, useVendor) } for _, path := range p.XTestImports { - download(path, stk, false) + if path == "C" { + continue + } + download(path, p, stk, useVendor) } } @@ -269,6 +330,12 @@ func downloadPackage(p *Package) error { repo, rootPath string err error ) + + security := secure + if *getInsecure { + security = insecure + } + if p.build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. vcs, rootPath, err = vcsForDir(p) @@ -278,10 +345,15 @@ func downloadPackage(p *Package) error { repo = "<local>" // should be unused; make distinctive // Double-check where it came from. - if *getU && vcs.remoteRepo != nil && !*getF { + if *getU && vcs.remoteRepo != nil { dir := filepath.Join(p.build.SrcRoot, rootPath) - if remote, err := vcs.remoteRepo(vcs, dir); err == nil { - if rr, err := repoRootForImportPath(p.ImportPath); err == nil { + remote, err := vcs.remoteRepo(vcs, dir) + if err != nil { + return err + } + repo = remote + if !*getF { + if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil { repo := rr.repo if rr.vcs.resolveRepo != nil { resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) @@ -289,7 +361,7 @@ func downloadPackage(p *Package) error { repo = resolved } } - if remote != repo { + if remote != repo && p.ImportComment != "" { return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote) } } @@ -298,12 +370,15 @@ func downloadPackage(p *Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - rr, err := repoRootForImportPath(p.ImportPath) + rr, err := repoRootForImportPath(p.ImportPath, security) if err != nil { return err } vcs, repo, rootPath = rr.vcs, rr.repo, rr.root } + if !vcs.isSecure(repo) && !*getInsecure { + return fmt.Errorf("cannot download, %v uses insecure protocol", repo) + } if p.build.SrcRoot == "" { // Package not found. Put in first directory of $GOPATH. diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go new file mode 100644 index 00000000000..77b2628982b --- /dev/null +++ b/libgo/go/cmd/go/go_test.go @@ -0,0 +1,2389 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main_test + +import ( + "bytes" + "flag" + "fmt" + "go/build" + "go/format" + "internal/testenv" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +var ( + canRun = true // whether we can run go or ./testgo + canRace = false // whether we can run the race detector + canCgo = false // whether we can use cgo + + exeSuffix string // ".exe" on Windows + + builder = testenv.Builder() + skipExternalBuilder = false // skip external tests on this builder +) + +func init() { + switch runtime.GOOS { + case "android", "nacl": + canRun = false + case "darwin": + switch runtime.GOARCH { + case "arm", "arm64": + canRun = false + } + } + + if strings.HasPrefix(builder+"-", "freebsd-arm-") { + skipExternalBuilder = true + canRun = false + } + + switch runtime.GOOS { + case "windows": + exeSuffix = ".exe" + } +} + +// The TestMain function creates a go command for testing purposes and +// deletes it after the tests have been run. +func TestMain(m *testing.M) { + flag.Parse() + + if canRun { + out, err := exec.Command("go", "build", "-tags", "testgo", "-o", "testgo"+exeSuffix).CombinedOutput() + if err != nil { + fmt.Fprintf(os.Stderr, "building testgo failed: %v\n%s", err, out) + os.Exit(2) + } + + if out, err := exec.Command("./testgo"+exeSuffix, "env", "CGO_ENABLED").Output(); err != nil { + fmt.Fprintf(os.Stderr, "running testgo failed: %v\n", err) + canRun = false + } else { + canCgo, err = strconv.ParseBool(strings.TrimSpace(string(out))) + if err != nil { + fmt.Fprintf(os.Stderr, "can't parse go env CGO_ENABLED output: %v\n", strings.TrimSpace(string(out))) + } + } + + switch runtime.GOOS { + case "linux", "darwin", "freebsd", "windows": + canRace = canCgo && runtime.GOARCH == "amd64" + } + + measureTick("./testgo" + exeSuffix) + } + + // Don't let these environment variables confuse the test. + os.Unsetenv("GOBIN") + os.Unsetenv("GOPATH") + + r := m.Run() + + if canRun { + os.Remove("testgo" + exeSuffix) + } + + os.Exit(r) +} + +// The length of an mtime tick on this system. This is an estimate of +// how long we need to sleep to ensure that the mtime of two files is +// different. +var mtimeTick time.Duration + +// measureTick sets mtimeTick by looking at the rounding of the mtime +// of a file. +func measureTick(path string) { + st, err := os.Stat(path) + if err != nil { + // Default to one second, the most conservative value. + mtimeTick = time.Second + return + } + mtime := st.ModTime() + t := time.Microsecond + for mtime.Round(t).Equal(mtime) && t < time.Second { + t *= 10 + } + mtimeTick = t +} + +// Manage a single run of the testgo binary. +type testgoData struct { + t *testing.T + temps []string + wd string + env []string + tempdir string + ran bool + inParallel bool + stdout, stderr bytes.Buffer +} + +// testgo sets up for a test that runs testgo. +func testgo(t *testing.T) *testgoData { + testenv.MustHaveGoBuild(t) + + if skipExternalBuilder { + t.Skip("skipping external tests on %s builder", builder) + } + + return &testgoData{t: t} +} + +// must gives a fatal error if err is not nil. +func (tg *testgoData) must(err error) { + if err != nil { + tg.t.Fatal(err) + } +} + +// check gives a test non-fatal error if err is not nil. +func (tg *testgoData) check(err error) { + if err != nil { + tg.t.Error(err) + } +} + +// parallel runs the test in parallel by calling t.Parallel. +func (tg *testgoData) parallel() { + if tg.ran { + tg.t.Fatal("internal testsuite error: call to parallel after run") + } + if tg.wd != "" { + tg.t.Fatal("internal testsuite error: call to parallel after cd") + } + for _, e := range tg.env { + if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { + val := e[strings.Index(e, "=")+1:] + if strings.HasPrefix(val, "testdata") || strings.HasPrefix(val, "./testdata") { + tg.t.Fatalf("internal testsuite error: call to parallel with testdata in environment (%s)", e) + } + } + } + tg.inParallel = true + tg.t.Parallel() +} + +// pwd returns the current directory. +func (tg *testgoData) pwd() string { + wd, err := os.Getwd() + if err != nil { + tg.t.Fatalf("could not get working directory: %v", err) + } + return wd +} + +// cd changes the current directory to the named directory. Note that +// using this means that the test must not be run in parallel with any +// other tests. +func (tg *testgoData) cd(dir string) { + if tg.inParallel { + tg.t.Fatal("internal testsuite error: changing directory when running in parallel") + } + if tg.wd == "" { + tg.wd = tg.pwd() + } + abs, err := filepath.Abs(dir) + tg.must(os.Chdir(dir)) + if err == nil { + tg.setenv("PWD", abs) + } +} + +// sleep sleeps for one tick, where a tick is a conservative estimate +// of how long it takes for a file modification to get a different +// mtime. +func (tg *testgoData) sleep() { + time.Sleep(mtimeTick) +} + +// setenv sets an environment variable to use when running the test go +// command. +func (tg *testgoData) setenv(name, val string) { + if tg.inParallel && (name == "GOROOT" || name == "GOPATH" || name == "GOBIN") && (strings.HasPrefix(val, "testdata") || strings.HasPrefix(val, "./testdata")) { + tg.t.Fatalf("internal testsuite error: call to setenv with testdata (%s=%s) after parallel", name, val) + } + tg.unsetenv(name) + tg.env = append(tg.env, name+"="+val) +} + +// unsetenv removes an environment variable. +func (tg *testgoData) unsetenv(name string) { + if tg.env == nil { + tg.env = append([]string(nil), os.Environ()...) + } + for i, v := range tg.env { + if strings.HasPrefix(v, name+"=") { + tg.env = append(tg.env[:i], tg.env[i+1:]...) + break + } + } +} + +// doRun runs the test go command, recording stdout and stderr and +// returning exit status. +func (tg *testgoData) doRun(args []string) error { + if !canRun { + panic("testgoData.doRun called but canRun false") + } + if tg.inParallel { + for _, arg := range args { + if strings.HasPrefix(arg, "testdata") || strings.HasPrefix(arg, "./testdata") { + tg.t.Fatal("internal testsuite error: parallel run using testdata") + } + } + } + tg.t.Logf("running testgo %v", args) + var prog string + if tg.wd == "" { + prog = "./testgo" + exeSuffix + } else { + prog = filepath.Join(tg.wd, "testgo"+exeSuffix) + } + cmd := exec.Command(prog, args...) + tg.stdout.Reset() + tg.stderr.Reset() + cmd.Stdout = &tg.stdout + cmd.Stderr = &tg.stderr + cmd.Env = tg.env + status := cmd.Run() + if tg.stdout.Len() > 0 { + tg.t.Log("standard output:") + tg.t.Log(tg.stdout.String()) + } + if tg.stderr.Len() > 0 { + tg.t.Log("standard error:") + tg.t.Log(tg.stderr.String()) + } + tg.ran = true + return status +} + +// run runs the test go command, and expects it to succeed. +func (tg *testgoData) run(args ...string) { + if status := tg.doRun(args); status != nil { + tg.t.Logf("go %v failed unexpectedly: %v", args, status) + tg.t.FailNow() + } +} + +// runFail runs the test go command, and expects it to fail. +func (tg *testgoData) runFail(args ...string) { + if status := tg.doRun(args); status == nil { + tg.t.Fatal("testgo succeeded unexpectedly") + } else { + tg.t.Log("testgo failed as expected:", status) + } +} + +// runGit runs a git command, and expects it to succeed. +func (tg *testgoData) runGit(dir string, args ...string) { + cmd := exec.Command("git", args...) + tg.stdout.Reset() + tg.stderr.Reset() + cmd.Stdout = &tg.stdout + cmd.Stderr = &tg.stderr + cmd.Dir = dir + cmd.Env = tg.env + status := cmd.Run() + if tg.stdout.Len() > 0 { + tg.t.Log("git standard output:") + tg.t.Log(tg.stdout.String()) + } + if tg.stderr.Len() > 0 { + tg.t.Log("git standard error:") + tg.t.Log(tg.stderr.String()) + } + if status != nil { + tg.t.Logf("git %v failed unexpectedly: %v", args, status) + tg.t.FailNow() + } +} + +// getStdout returns standard output of the testgo run as a string. +func (tg *testgoData) getStdout() string { + if !tg.ran { + tg.t.Fatal("internal testsuite error: stdout called before run") + } + return tg.stdout.String() +} + +// getStderr returns standard error of the testgo run as a string. +func (tg *testgoData) getStderr() string { + if !tg.ran { + tg.t.Fatal("internal testsuite error: stdout called before run") + } + return tg.stderr.String() +} + +// doGrepMatch looks for a regular expression in a buffer, and returns +// whether it is found. The regular expression is matched against +// each line separately, as with the grep command. +func (tg *testgoData) doGrepMatch(match string, b *bytes.Buffer) bool { + if !tg.ran { + tg.t.Fatal("internal testsuite error: grep called before run") + } + re := regexp.MustCompile(match) + for _, ln := range bytes.Split(b.Bytes(), []byte{'\n'}) { + if re.Match(ln) { + return true + } + } + return false +} + +// doGrep looks for a regular expression in a buffer and fails if it +// is not found. The name argument is the name of the output we are +// searching, "output" or "error". The msg argument is logged on +// failure. +func (tg *testgoData) doGrep(match string, b *bytes.Buffer, name, msg string) { + if !tg.doGrepMatch(match, b) { + tg.t.Log(msg) + tg.t.Logf("pattern %v not found in standard %s", match, name) + tg.t.FailNow() + } +} + +// grepStdout looks for a regular expression in the test run's +// standard output and fails, logging msg, if it is not found. +func (tg *testgoData) grepStdout(match, msg string) { + tg.doGrep(match, &tg.stdout, "output", msg) +} + +// grepStderr looks for a regular expression in the test run's +// standard error and fails, logging msg, if it is not found. +func (tg *testgoData) grepStderr(match, msg string) { + tg.doGrep(match, &tg.stderr, "error", msg) +} + +// grepBoth looks for a regular expression in the test run's standard +// output or stand error and fails, logging msg, if it is not found. +func (tg *testgoData) grepBoth(match, msg string) { + if !tg.doGrepMatch(match, &tg.stdout) && !tg.doGrepMatch(match, &tg.stderr) { + tg.t.Log(msg) + tg.t.Logf("pattern %v not found in standard output or standard error", match) + tg.t.FailNow() + } +} + +// doGrepNot looks for a regular expression in a buffer and fails if +// it is found. The name and msg arguments are as for doGrep. +func (tg *testgoData) doGrepNot(match string, b *bytes.Buffer, name, msg string) { + if tg.doGrepMatch(match, b) { + tg.t.Log(msg) + tg.t.Logf("pattern %v found unexpectedly in standard %s", match, name) + tg.t.FailNow() + } +} + +// grepStdoutNot looks for a regular expression in the test run's +// standard output and fails, logging msg, if it is found. +func (tg *testgoData) grepStdoutNot(match, msg string) { + tg.doGrepNot(match, &tg.stdout, "output", msg) +} + +// grepStderrNot looks for a regular expression in the test run's +// standard error and fails, logging msg, if it is found. +func (tg *testgoData) grepStderrNot(match, msg string) { + tg.doGrepNot(match, &tg.stderr, "error", msg) +} + +// grepBothNot looks for a regular expression in the test run's +// standard output or stand error and fails, logging msg, if it is +// found. +func (tg *testgoData) grepBothNot(match, msg string) { + if tg.doGrepMatch(match, &tg.stdout) || tg.doGrepMatch(match, &tg.stderr) { + tg.t.Log(msg) + tg.t.Fatalf("pattern %v found unexpectedly in standard output or standard error", match) + } +} + +// doGrepCount counts the number of times a regexp is seen in a buffer. +func (tg *testgoData) doGrepCount(match string, b *bytes.Buffer) int { + if !tg.ran { + tg.t.Fatal("internal testsuite error: doGrepCount called before run") + } + re := regexp.MustCompile(match) + c := 0 + for _, ln := range bytes.Split(b.Bytes(), []byte{'\n'}) { + if re.Match(ln) { + c++ + } + } + return c +} + +// grepCountStdout returns the number of times a regexp is seen in +// standard output. +func (tg *testgoData) grepCountStdout(match string) int { + return tg.doGrepCount(match, &tg.stdout) +} + +// grepCountStderr returns the number of times a regexp is seen in +// standard error. +func (tg *testgoData) grepCountStderr(match string) int { + return tg.doGrepCount(match, &tg.stderr) +} + +// grepCountBoth returns the number of times a regexp is seen in both +// standard output and standard error. +func (tg *testgoData) grepCountBoth(match string) int { + return tg.doGrepCount(match, &tg.stdout) + tg.doGrepCount(match, &tg.stderr) +} + +// creatingTemp records that the test plans to create a temporary file +// or directory. If the file or directory exists already, it will be +// removed. When the test completes, the file or directory will be +// removed if it exists. +func (tg *testgoData) creatingTemp(path string) { + if filepath.IsAbs(path) && !strings.HasPrefix(path, tg.tempdir) { + tg.t.Fatal("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path) + } + // If we have changed the working directory, make sure we have + // an absolute path, because we are going to change directory + // back before we remove the temporary. + if tg.wd != "" && !filepath.IsAbs(path) { + path = filepath.Join(tg.pwd(), path) + } + tg.must(os.RemoveAll(path)) + tg.temps = append(tg.temps, path) +} + +// makeTempdir makes a temporary directory for a run of testgo. If +// the temporary directory was already created, this does nothing. +func (tg *testgoData) makeTempdir() { + if tg.tempdir == "" { + var err error + tg.tempdir, err = ioutil.TempDir("", "gotest") + tg.must(err) + } +} + +// tempFile adds a temporary file for a run of testgo. +func (tg *testgoData) tempFile(path, contents string) { + tg.makeTempdir() + tg.must(os.MkdirAll(filepath.Join(tg.tempdir, filepath.Dir(path)), 0755)) + bytes := []byte(contents) + if strings.HasSuffix(path, ".go") { + formatted, err := format.Source(bytes) + if err == nil { + bytes = formatted + } + } + tg.must(ioutil.WriteFile(filepath.Join(tg.tempdir, path), bytes, 0644)) +} + +// tempDir adds a temporary directory for a run of testgo. +func (tg *testgoData) tempDir(path string) { + tg.makeTempdir() + if err := os.MkdirAll(filepath.Join(tg.tempdir, path), 0755); err != nil && !os.IsExist(err) { + tg.t.Fatal(err) + } +} + +// path returns the absolute pathname to file with the temporary +// directory. +func (tg *testgoData) path(name string) string { + if tg.tempdir == "" { + tg.t.Fatalf("internal testsuite error: path(%q) with no tempdir", name) + } + if name == "." { + return tg.tempdir + } + return filepath.Join(tg.tempdir, name) +} + +// mustNotExist fails if path exists. +func (tg *testgoData) mustNotExist(path string) { + if _, err := os.Stat(path); err == nil || !os.IsNotExist(err) { + tg.t.Fatalf("%s exists but should not (%v)", path, err) + } +} + +// wantExecutable fails with msg if path is not executable. +func (tg *testgoData) wantExecutable(path, msg string) { + if st, err := os.Stat(path); err != nil { + if !os.IsNotExist(err) { + tg.t.Log(err) + } + tg.t.Fatal(msg) + } else { + if runtime.GOOS != "windows" && st.Mode()&0111 == 0 { + tg.t.Fatalf("binary %s exists but is not executable", path) + } + } +} + +// wantArchive fails if path is not an archive. +func (tg *testgoData) wantArchive(path string) { + f, err := os.Open(path) + if err != nil { + tg.t.Fatal(err) + } + buf := make([]byte, 100) + io.ReadFull(f, buf) + f.Close() + if !bytes.HasPrefix(buf, []byte("!<arch>\n")) { + tg.t.Fatalf("file %s exists but is not an archive", path) + } +} + +// isStale returns whether pkg is stale. +func (tg *testgoData) isStale(pkg string) bool { + tg.run("list", "-f", "{{.Stale}}", pkg) + switch v := strings.TrimSpace(tg.getStdout()); v { + case "true": + return true + case "false": + return false + default: + tg.t.Fatalf("unexpected output checking staleness of package %v: %v", pkg, v) + panic("unreachable") + } +} + +// wantStale fails with msg if pkg is not stale. +func (tg *testgoData) wantStale(pkg, msg string) { + if !tg.isStale(pkg) { + tg.t.Fatal(msg) + } +} + +// wantNotStale fails with msg if pkg is stale. +func (tg *testgoData) wantNotStale(pkg, msg string) { + if tg.isStale(pkg) { + tg.t.Fatal(msg) + } +} + +// cleanup cleans up a test that runs testgo. +func (tg *testgoData) cleanup() { + if tg.wd != "" { + if err := os.Chdir(tg.wd); err != nil { + // We are unlikely to be able to continue. + fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err) + os.Exit(2) + } + } + for _, path := range tg.temps { + tg.check(os.RemoveAll(path)) + } + if tg.tempdir != "" { + tg.check(os.RemoveAll(tg.tempdir)) + } +} + +// resetReadOnlyFlagAll resets windows read-only flag +// set on path and any children it contains. +// The flag is set by git and has to be removed. +// os.Remove refuses to remove files with read-only flag set. +func (tg *testgoData) resetReadOnlyFlagAll(path string) { + fi, err := os.Stat(path) + if err != nil { + tg.t.Fatalf("resetReadOnlyFlagAll(%q) failed: %v", path, err) + } + if !fi.IsDir() { + err := os.Chmod(path, 0666) + if err != nil { + tg.t.Fatalf("resetReadOnlyFlagAll(%q) failed: %v", path, err) + } + } + fd, err := os.Open(path) + if err != nil { + tg.t.Fatalf("resetReadOnlyFlagAll(%q) failed: %v", path, err) + } + defer fd.Close() + names, _ := fd.Readdirnames(-1) + for _, name := range names { + tg.resetReadOnlyFlagAll(path + string(filepath.Separator) + name) + } +} + +// failSSH puts an ssh executable in the PATH that always fails. +// This is to stub out uses of ssh by go get. +func (tg *testgoData) failSSH() { + wd, err := os.Getwd() + if err != nil { + tg.t.Fatal(err) + } + fail := filepath.Join(wd, "testdata/failssh") + tg.setenv("PATH", fmt.Sprintf("%v%c%v", fail, filepath.ListSeparator, os.Getenv("PATH"))) +} + +func TestFileLineInErrorMessages(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("err.go", `package main; import "bar"`) + path := tg.path("err.go") + tg.runFail("run", path) + shortPath := path + if rel, err := filepath.Rel(tg.pwd(), path); err == nil && len(rel) < len(path) { + shortPath = rel + } + tg.grepStderr("^"+regexp.QuoteMeta(shortPath)+":", "missing file:line in error message") +} + +func TestProgramNameInCrashMessages(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("triv.go", `package main; func main() {}`) + tg.runFail("build", "-ldflags", "-crash_for_testing", tg.path("triv.go")) + tg.grepStderr(`[/\\]tool[/\\].*[/\\]link`, "missing linker name in error message") +} + +func TestBrokenTestsWithoutTestFunctionsAllFail(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("test", "./testdata/src/badtest/...") + tg.grepBothNot("^ok", "test passed unexpectedly") + tg.grepBoth("FAIL.*badtest/badexec", "test did not run everything") + tg.grepBoth("FAIL.*badtest/badsyntax", "test did not run everything") + tg.grepBoth("FAIL.*badtest/badvar", "test did not run everything") +} + +func TestGoBuildDashAInDevBranch(t *testing.T) { + if testing.Short() { + t.Skip("don't rebuild the standard library in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("install", "math") // should be up to date already but just in case + tg.setenv("TESTGO_IS_GO_RELEASE", "0") + tg.run("build", "-v", "-a", "math") + tg.grepStderr("runtime", "testgo build -a math in dev branch DID NOT build runtime, but should have") +} + +func TestGoBuilDashAInReleaseBranch(t *testing.T) { + if testing.Short() { + t.Skip("don't rebuild the standard library in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("install", "math") // should be up to date already but just in case + tg.setenv("TESTGO_IS_GO_RELEASE", "1") + tg.run("build", "-v", "-a", "math") + tg.grepStderr("runtime", "testgo build -a math in dev branch did not build runtime, but should have") +} + +func TestGoInstallCleansUpAfterGoBuild(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/mycmd/main.go", `package main; func main(){}`) + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("src/mycmd")) + + doesNotExist := func(file, msg string) { + if _, err := os.Stat(file); err == nil { + t.Fatal(msg) + } else if !os.IsNotExist(err) { + t.Fatal(msg, "error:", err) + } + } + + tg.run("build") + tg.wantExecutable("mycmd"+exeSuffix, "testgo build did not write command binary") + tg.run("install") + doesNotExist("mycmd"+exeSuffix, "testgo install did not remove command binary") + tg.run("build") + tg.wantExecutable("mycmd"+exeSuffix, "testgo build did not write command binary (second time)") + // Running install with arguments does not remove the target, + // even in the same directory. + tg.run("install", "mycmd") + tg.wantExecutable("mycmd"+exeSuffix, "testgo install mycmd removed command binary when run in mycmd") + tg.run("build") + tg.wantExecutable("mycmd"+exeSuffix, "testgo build did not write command binary (third time)") + // And especially not outside the directory. + tg.cd(tg.path(".")) + if data, err := ioutil.ReadFile("src/mycmd/mycmd" + exeSuffix); err != nil { + t.Fatal("could not read file:", err) + } else { + if err := ioutil.WriteFile("mycmd"+exeSuffix, data, 0555); err != nil { + t.Fatal("could not write file:", err) + } + } + tg.run("install", "mycmd") + tg.wantExecutable("src/mycmd/mycmd"+exeSuffix, "testgo install mycmd removed command binary from its source dir when run outside mycmd") + tg.wantExecutable("mycmd"+exeSuffix, "testgo install mycmd removed command binary from current dir when run outside mycmd") +} + +func TestGoInstallRebuildsStalePackagesInOtherGOPATH(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("d1/src/p1/p1.go", `package p1 + import "p2" + func F() { p2.F() }`) + tg.tempFile("d2/src/p2/p2.go", `package p2 + func F() {}`) + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", tg.path("d1")+sep+tg.path("d2")) + tg.run("install", "p1") + tg.wantNotStale("p1", "./testgo list mypkg claims p1 is stale, incorrectly") + tg.wantNotStale("p2", "./testgo list mypkg claims p2 is stale, incorrectly") + tg.sleep() + if f, err := os.OpenFile(tg.path("d2/src/p2/p2.go"), os.O_WRONLY|os.O_APPEND, 0); err != nil { + t.Fatal(err) + } else if _, err = f.WriteString(`func G() {}`); err != nil { + t.Fatal(err) + } else { + tg.must(f.Close()) + } + tg.wantStale("p2", "./testgo list mypkg claims p2 is NOT stale, incorrectly") + tg.wantStale("p1", "./testgo list mypkg claims p1 is NOT stale, incorrectly") + + tg.run("install", "p1") + tg.wantNotStale("p2", "./testgo list mypkg claims p2 is stale after reinstall, incorrectly") + tg.wantNotStale("p1", "./testgo list mypkg claims p1 is stale after reinstall, incorrectly") +} + +func TestGoInstallDetectsRemovedFiles(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/mypkg/x.go", `package mypkg`) + tg.tempFile("src/mypkg/y.go", `package mypkg`) + tg.tempFile("src/mypkg/z.go", `// +build missingtag + + package mypkg`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("install", "mypkg") + tg.wantNotStale("mypkg", "./testgo list mypkg claims mypkg is stale, incorrectly") + // z.go was not part of the build; removing it is okay. + tg.must(os.Remove(tg.path("src/mypkg/z.go"))) + tg.wantNotStale("mypkg", "./testgo list mypkg claims mypkg is stale after removing z.go; should not be stale") + // y.go was part of the package; removing it should be detected. + tg.must(os.Remove(tg.path("src/mypkg/y.go"))) + tg.wantStale("mypkg", "./testgo list mypkg claims mypkg is NOT stale after removing y.go; should be stale") +} + +func TestGoInstallErrorOnCrossCompileToBin(t *testing.T) { + if testing.Short() { + t.Skip("don't install into GOROOT in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/mycmd/x.go", `package main + func main() {}`) + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("src/mycmd")) + + tg.run("build", "mycmd") + + goarch := "386" + if runtime.GOARCH == "386" { + goarch = "amd64" + } + tg.setenv("GOOS", "linux") + tg.setenv("GOARCH", goarch) + tg.run("install", "mycmd") + tg.setenv("GOBIN", tg.path(".")) + tg.runFail("install", "mycmd") + tg.run("install", "cmd/pack") +} + +func TestGoInstallDetectsRemovedFilesInPackageMain(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/mycmd/x.go", `package main + func main() {}`) + tg.tempFile("src/mycmd/y.go", `package main`) + tg.tempFile("src/mycmd/z.go", `// +build missingtag + + package main`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("install", "mycmd") + tg.wantNotStale("mycmd", "./testgo list mypkg claims mycmd is stale, incorrectly") + // z.go was not part of the build; removing it is okay. + tg.must(os.Remove(tg.path("src/mycmd/z.go"))) + tg.wantNotStale("mycmd", "./testgo list mycmd claims mycmd is stale after removing z.go; should not be stale") + // y.go was part of the package; removing it should be detected. + tg.must(os.Remove(tg.path("src/mycmd/y.go"))) + tg.wantStale("mycmd", "./testgo list mycmd claims mycmd is NOT stale after removing y.go; should be stale") +} + +func testLocalRun(tg *testgoData, exepath, local, match string) { + out, err := exec.Command(exepath).Output() + if err != nil { + tg.t.Fatalf("error running %v: %v", exepath, err) + } + if !regexp.MustCompile(match).Match(out) { + tg.t.Log(string(out)) + tg.t.Errorf("testdata/%s/easy.go did not generate expected output", local) + } +} + +func testLocalEasy(tg *testgoData, local string) { + exepath := "./easy" + exeSuffix + tg.creatingTemp(exepath) + tg.run("build", "-o", exepath, filepath.Join("testdata", local, "easy.go")) + testLocalRun(tg, exepath, local, `(?m)^easysub\.Hello`) +} + +func testLocalEasySub(tg *testgoData, local string) { + exepath := "./easysub" + exeSuffix + tg.creatingTemp(exepath) + tg.run("build", "-o", exepath, filepath.Join("testdata", local, "easysub", "main.go")) + testLocalRun(tg, exepath, local, `(?m)^easysub\.Hello`) +} + +func testLocalHard(tg *testgoData, local string) { + exepath := "./hard" + exeSuffix + tg.creatingTemp(exepath) + tg.run("build", "-o", exepath, filepath.Join("testdata", local, "hard.go")) + testLocalRun(tg, exepath, local, `(?m)^sub\.Hello`) +} + +func testLocalInstall(tg *testgoData, local string) { + tg.runFail("install", filepath.Join("testdata", local, "easy.go")) +} + +func TestLocalImportsEasy(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + testLocalEasy(tg, "local") +} + +func TestLocalImportsEasySub(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + testLocalEasySub(tg, "local") +} + +func TestLocalImportsHard(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + testLocalHard(tg, "local") +} + +func TestLocalImportsGoInstallShouldFail(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + testLocalInstall(tg, "local") +} + +const badDirName = `#$%:, &()*;<=>?\^{}` + +func copyBad(tg *testgoData) { + if runtime.GOOS == "windows" { + tg.t.Skipf("skipping test because %q is an invalid directory name", badDirName) + } + + tg.must(filepath.Walk("testdata/local", + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + var data []byte + data, err = ioutil.ReadFile(path) + if err != nil { + return err + } + newpath := strings.Replace(path, "local", badDirName, 1) + tg.tempFile(newpath, string(data)) + return nil + })) + tg.cd(tg.path(".")) +} + +func TestBadImportsEasy(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + copyBad(tg) + testLocalEasy(tg, badDirName) +} + +func TestBadImportsEasySub(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + copyBad(tg) + testLocalEasySub(tg, badDirName) +} + +func TestBadImportsHard(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + copyBad(tg) + testLocalHard(tg, badDirName) +} + +func TestBadImportsGoInstallShouldFail(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + copyBad(tg) + testLocalInstall(tg, badDirName) +} + +func TestInternalPackagesInGOROOTAreRespected(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("build", "-v", "./testdata/testinternal") + tg.grepBoth("use of internal package not allowed", "wrong error message for testdata/testinternal") +} + +func TestInternalPackagesOutsideGOROOTAreRespected(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("build", "-v", "./testdata/testinternal2") + tg.grepBoth("use of internal package not allowed", "wrote error message for testdata/testinternal2") +} + +func testMove(t *testing.T, vcs, url, base, config string) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-d", url) + tg.run("get", "-d", "-u", url) + switch vcs { + case "svn": + // SVN doesn't believe in text files so we can't just edit the config. + // Check out a different repo into the wrong place. + tg.must(os.RemoveAll(tg.path("src/code.google.com/p/rsc-svn"))) + tg.run("get", "-d", "-u", "code.google.com/p/rsc-svn2/trunk") + tg.must(os.Rename(tg.path("src/code.google.com/p/rsc-svn2"), tg.path("src/code.google.com/p/rsc-svn"))) + default: + path := tg.path(filepath.Join("src", config)) + data, err := ioutil.ReadFile(path) + tg.must(err) + data = bytes.Replace(data, []byte(base), []byte(base+"XXX"), -1) + tg.must(ioutil.WriteFile(path, data, 0644)) + } + if vcs == "git" { + // git will ask for a username and password when we + // run go get -d -f -u. An empty username and + // password will work. Prevent asking by setting + // GIT_ASKPASS. + tg.creatingTemp("sink" + exeSuffix) + tg.tempFile("src/sink/sink.go", `package main; func main() {}`) + tg.run("build", "-o", "sink"+exeSuffix, "sink") + tg.setenv("GIT_ASKPASS", filepath.Join(tg.pwd(), "sink"+exeSuffix)) + } + tg.runFail("get", "-d", "-u", url) + tg.grepStderr("is a custom import path for", "go get -d -u "+url+" failed for wrong reason") + tg.runFail("get", "-d", "-f", "-u", url) + tg.grepStderr("validating server certificate|not found", "go get -d -f -u "+url+" failed for wrong reason") +} + +func TestInternalPackageErrorsAreHandled(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("list", "./testdata/testinternal3") +} + +func TestInternalCache(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/testinternal4")) + tg.runFail("build", "p") + tg.grepStderr("internal", "did not fail to build p") +} + +func TestMoveGit(t *testing.T) { + testMove(t, "git", "rsc.io/pdf", "pdf", "rsc.io/pdf/.git/config") +} + +// TODO(rsc): Set up a test case on bitbucket for hg. +// func TestMoveHG(t *testing.T) { +// testMove(t, "hg", "rsc.io/x86/x86asm", "x86", "rsc.io/x86/.hg/hgrc") +// } + +// TODO(rsc): Set up a test case on SourceForge (?) for svn. +// func testMoveSVN(t *testing.T) { +// testMove(t, "svn", "code.google.com/p/rsc-svn/trunk", "-", "-") +// } + +func TestImportCommandMatch(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) + tg.run("build", "./testdata/importcom/works.go") +} + +func TestImportCommentMismatch(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) + tg.runFail("build", "./testdata/importcom/wrongplace.go") + tg.grepStderr(`wrongplace expects import "my/x"`, "go build did not mention incorrect import") +} + +func TestImportCommentSyntaxError(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) + tg.runFail("build", "./testdata/importcom/bad.go") + tg.grepStderr("cannot parse import comment", "go build did not mention syntax error") +} + +func TestImportCommentConflict(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/importcom")) + tg.runFail("build", "./testdata/importcom/conflict.go") + tg.grepStderr("found import comments", "go build did not mention comment conflict") +} + +// cmd/go: custom import path checking should not apply to github.com/xxx/yyy. +func TestIssue10952(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + if _, err := exec.LookPath("git"); err != nil { + t.Skip("skipping because git binary not found") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + const importPath = "github.com/zombiezen/go-get-issue-10952" + tg.run("get", "-d", "-u", importPath) + repoDir := tg.path("src/" + importPath) + defer tg.resetReadOnlyFlagAll(repoDir) + tg.runGit(repoDir, "remote", "set-url", "origin", "https://"+importPath+".git") + tg.run("get", "-d", "-u", importPath) +} + +func TestDisallowedCSourceFiles(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("build", "badc") + tg.grepStderr("C source files not allowed", "go test did not say C source files not allowed") +} + +func TestErrorMessageForSyntaxErrorInTestGoFileSaysFAIL(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("test", "syntaxerror") + tg.grepStderr("FAIL", "go test did not say FAIL") +} + +func TestWildcardsDoNotLookInUselessDirectories(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("list", "...") + tg.grepBoth("badpkg", "go list ... failure does not mention badpkg") + tg.run("list", "m...") +} + +func TestRelativeImportsGoTest(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "./testdata/testimport") +} + +func TestRelativeImportsGoTestDashI(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "-i", "./testdata/testimport") +} + +func TestRelativeImportsInCommandLinePackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + files, err := filepath.Glob("./testdata/testimport/*.go") + tg.must(err) + tg.run(append([]string{"test"}, files...)...) +} + +func TestVersionControlErrorMessageIncludesCorrectDirectory(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/shadow/root1")) + tg.runFail("get", "-u", "foo") + + // TODO(iant): We should not have to use strconv.Quote here. + // The code in vcs.go should be changed so that it is not required. + quoted := strconv.Quote(filepath.Join("testdata", "shadow", "root1", "src", "foo")) + quoted = quoted[1 : len(quoted)-1] + + tg.grepStderr(regexp.QuoteMeta(quoted), "go get -u error does not mention shadow/root1/src/foo") +} + +func TestInstallFailsWithNoBuildableFiles(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("CGO_ENABLED", "0") + tg.runFail("install", "cgotest") + tg.grepStderr("no buildable Go source files", "go install cgotest did not report 'no buildable Go Source files'") +} + +// Test that without $GOBIN set, binaries get installed +// into the GOPATH bin directory. +func TestInstallIntoGOPATH(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.creatingTemp("testdata/bin/go-cmd-test" + exeSuffix) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("install", "go-cmd-test") + tg.wantExecutable("testdata/bin/go-cmd-test"+exeSuffix, "go install go-cmd-test did not write to testdata/bin/go-cmd-test") +} + +func TestPackageMainTestImportsArchiveNotBinary(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + gobin := filepath.Join(tg.pwd(), "testdata", "bin") + tg.creatingTemp(gobin) + tg.setenv("GOBIN", gobin) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.must(os.Chtimes("./testdata/src/main_test/m.go", time.Now(), time.Now())) + tg.sleep() + tg.run("test", "main_test") + tg.run("install", "main_test") + tg.wantNotStale("main_test", "after go install, main listed as stale") + tg.run("test", "main_test") +} + +// With $GOBIN set, binaries get installed to $GOBIN. +func TestInstallIntoGOBIN(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + gobin := filepath.Join(tg.pwd(), "testdata", "bin1") + tg.creatingTemp(gobin) + tg.setenv("GOBIN", gobin) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("install", "go-cmd-test") + tg.wantExecutable("testdata/bin1/go-cmd-test"+exeSuffix, "go install go-cmd-test did not write to testdata/bin1/go-cmd-test") +} + +// Issue 11065 +func TestInstallToCurrentDirectoryCreatesExecutable(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + pkg := filepath.Join(tg.pwd(), "testdata", "src", "go-cmd-test") + tg.creatingTemp(filepath.Join(pkg, "go-cmd-test"+exeSuffix)) + tg.setenv("GOBIN", pkg) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.cd(pkg) + tg.run("install") + tg.wantExecutable("go-cmd-test"+exeSuffix, "go install did not write to current directory") +} + +// Without $GOBIN set, installing a program outside $GOPATH should fail +// (there is nowhere to install it). +func TestInstallWithoutDestinationFails(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("install", "testdata/src/go-cmd-test/helloworld.go") + tg.grepStderr("no install location for .go files listed on command line", "wrong error") +} + +// With $GOBIN set, should install there. +func TestInstallToGOBINCommandLinePackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + gobin := filepath.Join(tg.pwd(), "testdata", "bin1") + tg.creatingTemp(gobin) + tg.setenv("GOBIN", gobin) + tg.run("install", "testdata/src/go-cmd-test/helloworld.go") + tg.wantExecutable("testdata/bin1/helloworld"+exeSuffix, "go install testdata/src/go-cmd-test/helloworld.go did not write testdata/bin1/helloworld") +} + +func TestGodocInstalls(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + // godoc installs into GOBIN + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("gobin") + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GOBIN", tg.path("gobin")) + tg.run("get", "golang.org/x/tools/cmd/godoc") + tg.wantExecutable(tg.path("gobin/godoc"), "did not install godoc to $GOBIN") + tg.unsetenv("GOBIN") + + // godoc installs into GOROOT + goroot := runtime.GOROOT() + tg.setenv("GOROOT", goroot) + tg.check(os.RemoveAll(filepath.Join(goroot, "bin", "godoc"))) + tg.run("install", "golang.org/x/tools/cmd/godoc") + tg.wantExecutable(filepath.Join(goroot, "bin", "godoc"), "did not install godoc to $GOROOT/bin") +} + +func TestGoGetNonPkg(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.tempDir("gobin") + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GOBIN", tg.path("gobin")) + tg.runFail("get", "-d", "golang.org/x/tools") + tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") + tg.runFail("get", "-d", "-u", "golang.org/x/tools") + tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") + tg.runFail("get", "-d", "golang.org/x/tools") + tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") +} + +func TestInstalls(t *testing.T) { + if testing.Short() { + t.Skip("don't install into GOROOT in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("gobin") + tg.setenv("GOPATH", tg.path(".")) + goroot := runtime.GOROOT() + tg.setenv("GOROOT", goroot) + + // cmd/fix installs into tool + tg.run("env", "GOOS") + goos := strings.TrimSpace(tg.getStdout()) + tg.setenv("GOOS", goos) + tg.run("env", "GOARCH") + goarch := strings.TrimSpace(tg.getStdout()) + tg.setenv("GOARCH", goarch) + fixbin := filepath.Join(goroot, "pkg", "tool", goos+"_"+goarch, "fix") + exeSuffix + tg.must(os.RemoveAll(fixbin)) + tg.run("install", "cmd/fix") + tg.wantExecutable(fixbin, "did not install cmd/fix to $GOROOT/pkg/tool") + tg.must(os.Remove(fixbin)) + tg.setenv("GOBIN", tg.path("gobin")) + tg.run("install", "cmd/fix") + tg.wantExecutable(fixbin, "did not install cmd/fix to $GOROOT/pkg/tool with $GOBIN set") + tg.unsetenv("GOBIN") + + // gopath program installs into GOBIN + tg.tempFile("src/progname/p.go", `package main; func main() {}`) + tg.setenv("GOBIN", tg.path("gobin")) + tg.run("install", "progname") + tg.unsetenv("GOBIN") + tg.wantExecutable(tg.path("gobin/progname")+exeSuffix, "did not install progname to $GOBIN/progname") + + // gopath program installs into GOPATH/bin + tg.run("install", "progname") + tg.wantExecutable(tg.path("bin/progname")+exeSuffix, "did not install progname to $GOPATH/bin/progname") +} + +func TestRejectRelativeDotPathInGOPATHCommandLinePackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", ".") + tg.runFail("build", "testdata/src/go-cmd-test/helloworld.go") + tg.grepStderr("GOPATH entry is relative", "expected an error message rejecting relative GOPATH entries") +} + +func TestRejectRelativePathsInGOPATH(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", sep+filepath.Join(tg.pwd(), "testdata")+sep+".") + tg.runFail("build", "go-cmd-test") + tg.grepStderr("GOPATH entry is relative", "expected an error message rejecting relative GOPATH entries") +} + +func TestRejectRelativePathsInGOPATHCommandLinePackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", "testdata") + tg.runFail("build", "testdata/src/go-cmd-test/helloworld.go") + tg.grepStderr("GOPATH entry is relative", "expected an error message rejecting relative GOPATH entries") +} + +// Issue 4104. +func TestGoTestWithPackageListedMultipleTimes(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.run("test", "errors", "errors", "errors", "errors", "errors") + if strings.Index(strings.TrimSpace(tg.getStdout()), "\n") != -1 { + t.Error("go test errors errors errors errors errors tested the same package multiple times") + } +} + +func TestGoListHasAConsistentOrder(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("list", "std") + first := tg.getStdout() + tg.run("list", "std") + if first != tg.getStdout() { + t.Error("go list std ordering is inconsistent") + } +} + +func TestGoListStdDoesNotIncludeCommands(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("list", "std") + tg.grepStdoutNot("cmd/", "go list std shows commands") +} + +func TestGoListCmdOnlyShowsCommands(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("list", "cmd") + out := strings.TrimSpace(tg.getStdout()) + for _, line := range strings.Split(out, "\n") { + if strings.Index(line, "cmd/") == -1 { + t.Error("go list cmd shows non-commands") + break + } + } +} + +// Issue 4096. Validate the output of unsuccessful go install foo/quxx. +func TestUnsuccessfulGoInstallShouldMentionMissingPackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(`cannot find package "foo/quxx" in any of`) != 1 { + t.Error(`go install foo/quxx expected error: .*cannot find package "foo/quxx" in any of`) + } +} + +func TestGOROOTSearchFailureReporting(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(regexp.QuoteMeta(filepath.Join("foo", "quxx"))+` \(from \$GOROOT\)$`) != 1 { + t.Error(`go install foo/quxx expected error: .*foo/quxx (from $GOROOT)`) + } +} + +func TestMultipleGOPATHEntriesReportedSeparately(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata", "a")+sep+filepath.Join(tg.pwd(), "testdata", "b")) + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(`testdata[/\\].[/\\]src[/\\]foo[/\\]quxx`) != 2 { + t.Error(`go install foo/quxx expected error: .*testdata/a/src/foo/quxx (from $GOPATH)\n.*testdata/b/src/foo/quxx`) + } +} + +// Test (from $GOPATH) annotation is reported for the first GOPATH entry, +func TestMentionGOPATHInFirstGOPATHEntry(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata", "a")+sep+filepath.Join(tg.pwd(), "testdata", "b")) + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(regexp.QuoteMeta(filepath.Join("testdata", "a", "src", "foo", "quxx"))+` \(from \$GOPATH\)$`) != 1 { + t.Error(`go install foo/quxx expected error: .*testdata/a/src/foo/quxx (from $GOPATH)`) + } +} + +// but not on the second. +func TestMentionGOPATHNotOnSecondEntry(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata", "a")+sep+filepath.Join(tg.pwd(), "testdata", "b")) + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(regexp.QuoteMeta(filepath.Join("testdata", "b", "src", "foo", "quxx"))+`$`) != 1 { + t.Error(`go install foo/quxx expected error: .*testdata/b/src/foo/quxx`) + } +} + +// Test missing GOPATH is reported. +func TestMissingGOPATHIsReported(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", "") + tg.runFail("install", "foo/quxx") + if tg.grepCountBoth(`\(\$GOPATH not set\)$`) != 1 { + t.Error(`go install foo/quxx expected error: ($GOPATH not set)`) + } +} + +// Issue 4186. go get cannot be used to download packages to $GOROOT. +// Test that without GOPATH set, go get should fail. +func TestWithoutGOPATHGoGetFails(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", "") + tg.setenv("GOROOT", tg.path(".")) + tg.runFail("get", "-d", "golang.org/x/codereview/cmd/hgpatch") +} + +// Test that with GOPATH=$GOROOT, go get should fail. +func TestWithGOPATHEqualsGOROOTGoGetFails(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GOROOT", tg.path(".")) + tg.runFail("get", "-d", "golang.org/x/codereview/cmd/hgpatch") +} + +func TestLdflagsArgumentsWithSpacesIssue3941(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("main.go", `package main + var extern string + func main() { + println(extern) + }`) + tg.run("run", "-ldflags", `-X main.extern "hello world"`, tg.path("main.go")) + tg.grepStderr("^hello world", `ldflags -X main.extern 'hello world' failed`) +} + +func TestGoTestCpuprofileLeavesBinaryBehind(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.cd(tg.path(".")) + tg.run("test", "-cpuprofile", "errors.prof", "errors") + tg.wantExecutable("errors.test"+exeSuffix, "go test -cpuprofile did not create errors.test") +} + +func TestGoTestCpuprofileDashOControlsBinaryLocation(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.cd(tg.path(".")) + tg.run("test", "-cpuprofile", "errors.prof", "-o", "myerrors.test"+exeSuffix, "errors") + tg.wantExecutable("myerrors.test"+exeSuffix, "go test -cpuprofile -o myerrors.test did not create myerrors.test") +} + +func TestGoTestDashCDashOControlsBinaryLocation(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.run("test", "-c", "-o", tg.path("myerrors.test"+exeSuffix), "errors") + tg.wantExecutable(tg.path("myerrors.test"+exeSuffix), "go test -c -o myerrors.test did not create myerrors.test") +} + +func TestGoTestDashOWritesBinary(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.run("test", "-o", tg.path("myerrors.test"+exeSuffix), "errors") + tg.wantExecutable(tg.path("myerrors.test"+exeSuffix), "go test -o myerrors.test did not create myerrors.test") +} + +// Issue 4568. +func TestSymlinksDoNotConfuseGoList(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping symlink test on %s", runtime.GOOS) + } + + tg := testgo(t) + defer tg.cleanup() + tg.tempDir("src") + tg.must(os.Symlink(tg.path("."), tg.path("src/dir1"))) + tg.tempFile("src/dir1/p.go", "package p") + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("src")) + tg.run("list", "-f", "{{.Root}}", "dir1") + if strings.TrimSpace(tg.getStdout()) != tg.path(".") { + t.Error("confused by symlinks") + } +} + +// Issue 4515. +func TestInstallWithTags(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("bin") + tg.tempFile("src/example/a/main.go", `package main + func main() {}`) + tg.tempFile("src/example/b/main.go", `// +build mytag + + package main + func main() {}`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("install", "-tags", "mytag", "example/a", "example/b") + tg.wantExecutable(tg.path("bin/a"+exeSuffix), "go install example/a example/b did not install binaries") + tg.wantExecutable(tg.path("bin/b"+exeSuffix), "go install example/a example/b did not install binaries") + tg.must(os.Remove(tg.path("bin/a" + exeSuffix))) + tg.must(os.Remove(tg.path("bin/b" + exeSuffix))) + tg.run("install", "-tags", "mytag", "example/...") + tg.wantExecutable(tg.path("bin/a"+exeSuffix), "go install example/... did not install binaries") + tg.wantExecutable(tg.path("bin/b"+exeSuffix), "go install example/... did not install binaries") + tg.run("list", "-tags", "mytag", "example/b...") + if strings.TrimSpace(tg.getStdout()) != "example/b" { + t.Error("go list example/b did not find example/b") + } +} + +// Issue 4773 +func TestCaseCollisions(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src/example/a/pkg") + tg.tempDir("src/example/a/Pkg") + tg.tempDir("src/example/b") + tg.setenv("GOPATH", tg.path(".")) + tg.tempFile("src/example/a/a.go", `package p + import ( + _ "example/a/pkg" + _ "example/a/Pkg" + )`) + tg.tempFile("src/example/a/pkg/pkg.go", `package pkg`) + tg.tempFile("src/example/a/Pkg/pkg.go", `package pkg`) + tg.runFail("list", "example/a") + tg.grepStderr("case-insensitive import collision", "go list example/a did not report import collision") + tg.tempFile("src/example/b/file.go", `package b`) + tg.tempFile("src/example/b/FILE.go", `package b`) + f, err := os.Open(tg.path("src/example/b")) + tg.must(err) + names, err := f.Readdirnames(0) + tg.must(err) + tg.check(f.Close()) + args := []string{"list"} + if len(names) == 2 { + // case-sensitive file system, let directory read find both files + args = append(args, "example/b") + } else { + // case-insensitive file system, list files explicitly on command line + args = append(args, tg.path("src/example/b/file.go"), tg.path("src/example/b/FILE.go")) + } + tg.runFail(args...) + tg.grepStderr("case-insensitive file name collision", "go list example/b did not report file name collision") +} + +// Issue 8181. +func TestGoGetDashTIssue8181(t *testing.T) { + if testing.Short() { + t.Skip("skipping test that uses network in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-v", "-t", "github.com/rsc/go-get-issue-8181/a", "github.com/rsc/go-get-issue-8181/b") + tg.run("list", "...") + tg.grepStdout("x/build/cmd/cl", "missing expected x/build/cmd/cl") +} + +func TestIssue11307(t *testing.T) { + // go get -u was not working except in checkout directory + if testing.Short() { + t.Skip("skipping test that uses network in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "github.com/rsc/go-get-issue-11307") + tg.run("get", "-u", "github.com/rsc/go-get-issue-11307") // was failing +} + +func TestShadowingLogic(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + pwd := tg.pwd() + sep := string(filepath.ListSeparator) + tg.setenv("GOPATH", filepath.Join(pwd, "testdata", "shadow", "root1")+sep+filepath.Join(pwd, "testdata", "shadow", "root2")) + + // The math in root1 is not "math" because the standard math is. + tg.run("list", "-f", "({{.ImportPath}}) ({{.ConflictDir}})", "./testdata/shadow/root1/src/math") + pwdForwardSlash := strings.Replace(pwd, string(os.PathSeparator), "/", -1) + if !strings.HasPrefix(pwdForwardSlash, "/") { + pwdForwardSlash = "/" + pwdForwardSlash + } + // The output will have makeImportValid applies, but we only + // bother to deal with characters we might reasonably see. + pwdForwardSlash = strings.Replace(pwdForwardSlash, ":", "_", -1) + want := "(_" + pwdForwardSlash + "/testdata/shadow/root1/src/math) (" + filepath.Join(runtime.GOROOT(), "src", "math") + ")" + if strings.TrimSpace(tg.getStdout()) != want { + t.Error("shadowed math is not shadowed; looking for", want) + } + + // The foo in root1 is "foo". + tg.run("list", "-f", "({{.ImportPath}}) ({{.ConflictDir}})", "./testdata/shadow/root1/src/foo") + if strings.TrimSpace(tg.getStdout()) != "(foo) ()" { + t.Error("unshadowed foo is shadowed") + } + + // The foo in root2 is not "foo" because the foo in root1 got there first. + tg.run("list", "-f", "({{.ImportPath}}) ({{.ConflictDir}})", "./testdata/shadow/root2/src/foo") + want = "(_" + pwdForwardSlash + "/testdata/shadow/root2/src/foo) (" + filepath.Join(pwd, "testdata", "shadow", "root1", "src", "foo") + ")" + if strings.TrimSpace(tg.getStdout()) != want { + t.Error("shadowed foo is not shadowed; looking for", want) + } + + // The error for go install should mention the conflicting directory. + tg.runFail("install", "./testdata/shadow/root2/src/foo") + want = "go install: no install location for " + filepath.Join(pwd, "testdata", "shadow", "root2", "src", "foo") + ": hidden by " + filepath.Join(pwd, "testdata", "shadow", "root1", "src", "foo") + if strings.TrimSpace(tg.getStderr()) != want { + t.Error("wrong shadowed install error; looking for", want) + } +} + +// Only succeeds if source order is preserved. +func TestSourceFileNameOrderPreserved(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "testdata/example1_test.go", "testdata/example2_test.go") +} + +// Check that coverage analysis works at all. +// Don't worry about the exact numbers but require not 0.0%. +func checkCoverage(tg *testgoData, data string) { + if regexp.MustCompile(`[^0-9]0\.0%`).MatchString(data) { + tg.t.Error("some coverage results are 0.0%") + } + tg.t.Log(data) +} + +func TestCoverageRuns(t *testing.T) { + if testing.Short() { + t.Skip("don't build libraries for coverage in short mode") + } + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "-short", "-coverpkg=strings", "strings", "regexp") + data := tg.getStdout() + tg.getStderr() + tg.run("test", "-short", "-cover", "strings", "math", "regexp") + data += tg.getStdout() + tg.getStderr() + checkCoverage(tg, data) +} + +// Check that coverage analysis uses set mode. +func TestCoverageUsesSetMode(t *testing.T) { + if testing.Short() { + t.Skip("don't build libraries for coverage in short mode") + } + tg := testgo(t) + defer tg.cleanup() + tg.creatingTemp("testdata/cover.out") + tg.run("test", "-short", "-cover", "encoding/binary", "-coverprofile=testdata/cover.out") + data := tg.getStdout() + tg.getStderr() + if out, err := ioutil.ReadFile("testdata/cover.out"); err != nil { + t.Error(err) + } else { + if !bytes.Contains(out, []byte("mode: set")) { + t.Error("missing mode: set") + } + } + checkCoverage(tg, data) +} + +func TestCoverageUsesAtomicModeForRace(t *testing.T) { + if testing.Short() { + t.Skip("don't build libraries for coverage in short mode") + } + if !canRace { + t.Skip("skipping because race detector not supported") + } + + tg := testgo(t) + defer tg.cleanup() + tg.creatingTemp("testdata/cover.out") + tg.run("test", "-short", "-race", "-cover", "encoding/binary", "-coverprofile=testdata/cover.out") + data := tg.getStdout() + tg.getStderr() + if out, err := ioutil.ReadFile("testdata/cover.out"); err != nil { + t.Error(err) + } else { + if !bytes.Contains(out, []byte("mode: atomic")) { + t.Error("missing mode: atomic") + } + } + checkCoverage(tg, data) +} + +func TestCoverageUsesActualSettingToOverrideEvenForRace(t *testing.T) { + if testing.Short() { + t.Skip("don't build libraries for coverage in short mode") + } + if !canRace { + t.Skip("skipping because race detector not supported") + } + + tg := testgo(t) + defer tg.cleanup() + tg.creatingTemp("testdata/cover.out") + tg.run("test", "-short", "-race", "-cover", "encoding/binary", "-covermode=count", "-coverprofile=testdata/cover.out") + data := tg.getStdout() + tg.getStderr() + if out, err := ioutil.ReadFile("testdata/cover.out"); err != nil { + t.Error(err) + } else { + if !bytes.Contains(out, []byte("mode: count")) { + t.Error("missing mode: count") + } + } + checkCoverage(tg, data) +} + +func TestCoverageWithCgo(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "-short", "-cover", "./testdata/cgocover") + data := tg.getStdout() + tg.getStderr() + checkCoverage(tg, data) +} + +func TestCgoDependsOnSyscall(t *testing.T) { + if testing.Short() { + t.Skip("skipping test that removes $GOROOT/pkg/*_race in short mode") + } + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + if !canRace { + t.Skip("skipping because race detector not supported") + } + + tg := testgo(t) + defer tg.cleanup() + files, err := filepath.Glob(filepath.Join(runtime.GOROOT(), "pkg", "*_race")) + tg.must(err) + for _, file := range files { + tg.check(os.RemoveAll(file)) + } + tg.tempFile("src/foo/foo.go", ` + package foo + //#include <stdio.h> + import "C"`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("build", "-race", "foo") +} + +func TestCgoShowsFullPathNames(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/x/y/dirname/foo.go", ` + package foo + import "C" + func f() {`) + tg.setenv("GOPATH", tg.path(".")) + tg.runFail("build", "x/y/dirname") + tg.grepBoth("x/y/dirname", "error did not use full path") +} + +func TestCgoHandlesWlORIGIN(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/origin/origin.go", `package origin + // #cgo !darwin LDFLAGS: -Wl,-rpath -Wl,$ORIGIN + // void f(void) {} + import "C" + func f() { C.f() }`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("build", "origin") +} + +// "go test -c -test.bench=XXX errors" should not hang +func TestIssue6480(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.cd(tg.path(".")) + tg.run("test", "-c", "-test.bench=XXX", "errors") +} + +// cmd/cgo: undefined reference when linking a C-library using gccgo +func TestIssue7573(t *testing.T) { + if _, err := exec.LookPath("gccgo"); err != nil { + t.Skip("skipping because no gccgo compiler found") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/cgoref/cgoref.go", ` +package main +// #cgo LDFLAGS: -L alibpath -lalib +// void f(void) {} +import "C" + +func main() { C.f() }`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("build", "-n", "-compiler", "gccgo", "cgoref") + tg.grepStderr(`gccgo.*\-L alibpath \-lalib`, `no Go-inline "#cgo LDFLAGS:" ("-L alibpath -lalib") passed to gccgo linking stage`) +} + +func TestListTemplateCanUseContextFunction(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("list", "-f", "GOARCH: {{context.GOARCH}}") +} + +// cmd/go: "go test" should fail if package does not build +func TestIssue7108(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("test", "notest") +} + +// cmd/go: go test -a foo does not rebuild regexp. +func TestIssue6844(t *testing.T) { + if testing.Short() { + t.Skip("don't rebuild the standard libary in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + tg.creatingTemp("deps.test" + exeSuffix) + tg.run("test", "-x", "-a", "-c", "testdata/dep_test.go") + tg.grepStderr("regexp", "go test -x -a -c testdata/dep-test.go did not rebuild regexp") +} + +func TestBuildDashIInstallsDependencies(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/x/y/foo/foo.go", `package foo + func F() {}`) + tg.tempFile("src/x/y/bar/bar.go", `package bar + import "x/y/foo" + func F() { foo.F() }`) + tg.setenv("GOPATH", tg.path(".")) + + checkbar := func(desc string) { + tg.sleep() + tg.must(os.Chtimes(tg.path("src/x/y/foo/foo.go"), time.Now(), time.Now())) + tg.sleep() + tg.run("build", "-v", "-i", "x/y/bar") + tg.grepBoth("x/y/foo", "first build -i "+desc+" did not build x/y/foo") + tg.run("build", "-v", "-i", "x/y/bar") + tg.grepBothNot("x/y/foo", "second build -i "+desc+" built x/y/foo") + } + checkbar("pkg") + tg.creatingTemp("bar" + exeSuffix) + tg.tempFile("src/x/y/bar/bar.go", `package main + import "x/y/foo" + func main() { foo.F() }`) + checkbar("cmd") +} + +func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.runFail("build", "./testdata/testonly") + tg.grepStderr("no buildable Go", "go build ./testdata/testonly produced unexpected error") +} + +func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("test", "-c", "testcycle/p3") + tg.grepStderr("import cycle not allowed in test", "go test testcycle/p3 produced unexpected error") + + tg.runFail("test", "-c", "testcycle/q1") + tg.grepStderr("import cycle not allowed in test", "go test testcycle/q1 produced unexpected error") +} + +func TestGoTestFooTestWorks(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "testdata/standalone_test.go") +} + +func TestGoTestXtestonlyWorks(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("clean", "-i", "xtestonly") + tg.run("test", "xtestonly") +} + +func TestGoTestBuildsAnXtestContainingOnlyNonRunnableExamples(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "-v", "./testdata/norunexample") + tg.grepStdout("File with non-runnable example was built.", "file with non-runnable example was not built") +} + +func TestGoGenerateHandlesSimpleCommand(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping because windows has no echo command") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("generate", "./testdata/generate/test1.go") + tg.grepStdout("Success", "go generate ./testdata/generate/test1.go generated wrong output") +} + +func TestGoGenerateHandlesCommandAlias(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping because windows has no echo command") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("generate", "./testdata/generate/test2.go") + tg.grepStdout("Now is the time for all good men", "go generate ./testdata/generate/test2.go generated wrong output") +} + +func TestGoGenerateVariableSubstitution(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping because windows has no echo command") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("generate", "./testdata/generate/test3.go") + tg.grepStdout(runtime.GOARCH+" test3.go:7 pabc xyzp/test3.go/123", "go generate ./testdata/generate/test3.go generated wrong output") +} + +func TestGoGenerateRunFlag(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping because windows has no echo command") + } + + tg := testgo(t) + defer tg.cleanup() + tg.run("generate", "-run", "y.s", "./testdata/generate/test4.go") + tg.grepStdout("yes", "go generate -run yes ./testdata/generate/test4.go did not select yes") + tg.grepStdoutNot("no", "go generate -run yes ./testdata/generate/test4.go selected no") +} + +func TestGoGetCustomDomainWildcard(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-u", "rsc.io/pdf/...") + tg.wantExecutable(tg.path("bin/pdfpasswd"+exeSuffix), "did not build rsc/io/pdf/pdfpasswd") +} + +func TestGoGetInternalWildcard(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + // used to fail with errors about internal packages + tg.run("get", "github.com/rsc/go-get-issue-11960/...") +} + +func TestGoVetWithExternalTests(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "golang.org/x/tools/cmd/vet") + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("vet", "vetpkg") + tg.grepBoth("missing argument for Printf", "go vet vetpkg did not find missing argument for Printf") +} + +func TestGoVetWithTags(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "golang.org/x/tools/cmd/vet") + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("vet", "-tags", "tagtest", "vetpkg") + tg.grepBoth(`c\.go.*wrong number of args for format`, "go get vetpkg did not run scan tagged file") +} + +// Issue 9767. +func TestGoGetRscIoToolstash(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.tempDir("src/rsc.io") + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("src/rsc.io")) + tg.run("get", "./toolstash") +} + +// Test that you can not import a main package. +func TestIssue4210(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/x/main.go", `package main + var X int + func main() {}`) + tg.tempFile("src/y/main.go", `package main + import "fmt" + import xmain "x" + func main() { + fmt.Println(xmain.X) + }`) + tg.setenv("GOPATH", tg.path(".")) + tg.runFail("build", "y") + tg.grepBoth("is a program", `did not find expected error message ("is a program")`) +} + +func TestGoGetInsecure(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.failSSH() + + const repo = "wh3rd.net/git.git" + + // Try go get -d of HTTP-only repo (should fail). + tg.runFail("get", "-d", repo) + + // Try again with -insecure (should succeed). + tg.run("get", "-d", "-insecure", repo) + + // Try updating without -insecure (should fail). + tg.runFail("get", "-d", "-u", "-f", repo) +} + +func TestGoGetUpdateInsecure(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + + const repo = "github.com/golang/example" + + // Clone the repo via HTTP manually. + cmd := exec.Command("git", "clone", "-q", "http://"+repo, tg.path("src/"+repo)) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("cloning %v repo: %v\n%s", repo, err, out) + } + + // Update without -insecure should fail. + // Update with -insecure should succeed. + // We need -f to ignore import comments. + const pkg = repo + "/hello" + tg.runFail("get", "-d", "-u", "-f", pkg) + tg.run("get", "-d", "-u", "-f", "-insecure", pkg) +} + +func TestGoGetInsecureCustomDomain(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + + const repo = "wh3rd.net/repo" + tg.runFail("get", "-d", repo) + tg.run("get", "-d", "-insecure", repo) +} + +func TestIssue10193(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + tg.runFail("get", "code.google.com/p/rsc/pdf") + tg.grepStderr("is shutting down", "missed warning about code.google.com") +} + +func TestGoRunDirs(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.cd("testdata/rundir") + tg.runFail("run", "x.go", "sub/sub.go") + tg.grepStderr("named files must all be in one directory; have ./ and sub/", "wrong output") + tg.runFail("run", "sub/sub.go", "x.go") + tg.grepStderr("named files must all be in one directory; have sub/ and ./", "wrong output") +} + +func TestGoInstallPkgdir(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + pkg := tg.path(".") + tg.run("install", "-pkgdir", pkg, "errors") + _, err := os.Stat(filepath.Join(pkg, "errors.a")) + tg.must(err) + _, err = os.Stat(filepath.Join(pkg, "runtime.a")) + tg.must(err) +} + +func TestGoTestRaceInstallCgo(t *testing.T) { + switch sys := runtime.GOOS + "/" + runtime.GOARCH; sys { + case "darwin/amd64", "freebsd/amd64", "linux/amd64", "windows/amd64": + // ok + default: + t.Skip("no race detector on %s", sys) + } + + if !build.Default.CgoEnabled { + t.Skip("no race detector without cgo") + } + + // golang.org/issue/10500. + // This used to install a race-enabled cgo. + tg := testgo(t) + defer tg.cleanup() + tg.run("tool", "-n", "cgo") + cgo := strings.TrimSpace(tg.stdout.String()) + old, err := os.Stat(cgo) + tg.must(err) + tg.run("test", "-race", "-i", "runtime/race") + new, err := os.Stat(cgo) + tg.must(err) + if new.ModTime() != old.ModTime() { + t.Fatalf("go test -i runtime/race reinstalled cmd/cgo") + } +} + +func TestGoTestImportErrorStack(t *testing.T) { + const out = `package testdep/p1 (test) + imports testdep/p2 + imports testdep/p3: no buildable Go source files` + + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("test", "testdep/p1") + if !strings.Contains(tg.stderr.String(), out) { + t.Fatal("did not give full import stack:\n\n%s", tg.stderr.String()) + } +} + +func TestGoGetUpdate(t *testing.T) { + // golang.org/issue/9224. + // The recursive updating was trying to walk to + // former dependencies, not current ones. + + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + + rewind := func() { + tg.run("get", "github.com/rsc/go-get-issue-9224-cmd") + cmd := exec.Command("git", "reset", "--hard", "HEAD~") + cmd.Dir = tg.path("src/github.com/rsc/go-get-issue-9224-lib") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git: %v\n%s", err, out) + } + } + + rewind() + tg.run("get", "-u", "github.com/rsc/go-get-issue-9224-cmd") + + // Again with -d -u. + rewind() + tg.run("get", "-d", "-u", "github.com/rsc/go-get-issue-9224-cmd") +} + +func TestGoGetDomainRoot(t *testing.T) { + // golang.org/issue/9357. + // go get foo.io (not foo.io/subdir) was not working consistently. + + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + + // go-get-issue-9357.appspot.com is running + // the code at github.com/rsc/go-get-issue-9357, + // a trivial Go on App Engine app that serves a + // <meta> tag for the domain root. + tg.run("get", "-d", "go-get-issue-9357.appspot.com") + tg.run("get", "go-get-issue-9357.appspot.com") + tg.run("get", "-u", "go-get-issue-9357.appspot.com") + + tg.must(os.RemoveAll(tg.path("src/go-get-issue-9357.appspot.com"))) + tg.run("get", "go-get-issue-9357.appspot.com") + + tg.must(os.RemoveAll(tg.path("src/go-get-issue-9357.appspot.com"))) + tg.run("get", "-u", "go-get-issue-9357.appspot.com") +} + +func TestGoInstallShadowedGOPATH(t *testing.T) { + // golang.org/issue/3652. + // go get foo.io (not foo.io/subdir) was not working consistently. + + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path("gopath1")+string(filepath.ListSeparator)+tg.path("gopath2")) + + tg.tempDir("gopath1/src/test") + tg.tempDir("gopath2/src/test") + tg.tempFile("gopath2/src/test/main.go", "package main\nfunc main(){}\n") + + tg.cd(tg.path("gopath2/src/test")) + tg.runFail("install") + tg.grepStderr("no install location for.*gopath2.src.test: hidden by .*gopath1.src.test", "missing error") +} + +func TestIssue11709(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("run.go", ` + package main + import "os" + func main() { + if os.Getenv("TERM") != "" { + os.Exit(1) + } + }`) + tg.unsetenv("TERM") + tg.run("run", tg.path("run.go")) +} + +func TestIssue12096(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("test_test.go", ` + package main + import ("os"; "testing") + func TestEnv(t *testing.T) { + if os.Getenv("TERM") != "" { + t.Fatal("TERM is set") + } + }`) + tg.unsetenv("TERM") + tg.run("test", tg.path("test_test.go")) +} + +func TestGoBuildOutput(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + + tg.makeTempdir() + tg.cd(tg.path(".")) + + nonExeSuffix := ".exe" + if exeSuffix == ".exe" { + nonExeSuffix = "" + } + + tg.tempFile("x.go", "package main\nfunc main(){}\n") + tg.run("build", "x.go") + tg.wantExecutable("x"+exeSuffix, "go build x.go did not write x"+exeSuffix) + tg.must(os.Remove(tg.path("x" + exeSuffix))) + tg.mustNotExist("x" + nonExeSuffix) + + tg.run("build", "-o", "myprog", "x.go") + tg.mustNotExist("x") + tg.mustNotExist("x.exe") + tg.wantExecutable("myprog", "go build -o myprog x.go did not write myprog") + tg.mustNotExist("myprog.exe") + + tg.tempFile("p.go", "package p\n") + tg.run("build", "p.go") + tg.mustNotExist("p") + tg.mustNotExist("p.a") + tg.mustNotExist("p.o") + tg.mustNotExist("p.exe") + + tg.run("build", "-o", "p.a", "p.go") + tg.wantArchive("p.a") + + tg.run("build", "cmd/gofmt") + tg.wantExecutable("gofmt"+exeSuffix, "go build cmd/gofmt did not write gofmt"+exeSuffix) + tg.must(os.Remove(tg.path("gofmt" + exeSuffix))) + tg.mustNotExist("gofmt" + nonExeSuffix) + + tg.run("build", "-o", "mygofmt", "cmd/gofmt") + tg.wantExecutable("mygofmt", "go build -o mygofmt cmd/gofmt did not write mygofmt") + tg.mustNotExist("mygofmt.exe") + tg.mustNotExist("gofmt") + tg.mustNotExist("gofmt.exe") + + tg.run("build", "sync/atomic") + tg.mustNotExist("atomic") + tg.mustNotExist("atomic.exe") + + tg.run("build", "-o", "myatomic.a", "sync/atomic") + tg.wantArchive("myatomic.a") + tg.mustNotExist("atomic") + tg.mustNotExist("atomic.a") + tg.mustNotExist("atomic.exe") + + tg.runFail("build", "-o", "whatever", "cmd/gofmt", "sync/atomic") + tg.grepStderr("multiple packages", "did not reject -o with multiple packages") +} + +func TestGoBuildARM(t *testing.T) { + if testing.Short() { + t.Skip("skipping cross-compile in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + + tg.makeTempdir() + tg.cd(tg.path(".")) + + tg.setenv("GOARCH", "arm") + tg.setenv("GOOS", "linux") + tg.setenv("GOARM", "5") + tg.tempFile("hello.go", `package main + func main() {}`) + tg.run("build", "hello.go") + tg.grepStderrNot("unable to find math.a", "did not build math.a correctly") +} diff --git a/libgo/go/cmd/go/help.go b/libgo/go/cmd/go/help.go index c590fdb37fe..5dff2670f1b 100644 --- a/libgo/go/cmd/go/help.go +++ b/libgo/go/cmd/go/help.go @@ -11,7 +11,7 @@ var helpC = &Command{ There are two different ways to call between Go and C/C++ code. The first is the cgo tool, which is part of the Go distribution. For -information on how to use it see the cgo documentation (godoc cmd/cgo). +information on how to use it see the cgo documentation (go doc cmd/cgo). The second is the SWIG program, which is a general tool for interfacing between languages. For information on SWIG see @@ -47,7 +47,7 @@ environment variable (see 'go help gopath'). If no import paths are given, the action applies to the package in the current directory. -There are three reserved names for paths that should not be used +There are four reserved names for paths that should not be used for packages to be built with the go tool: - "main" denotes the top-level package in a stand-alone executable. @@ -59,6 +59,9 @@ system. - "std" is like all but expands to just the packages in the standard Go library. +- "cmd" expands to the Go repository's commands and their +internal libraries. + An import path is a pattern if it includes one or more "..." wildcards, each of which can match any string, including the empty string and strings containing slashes. Such a pattern expands to all package @@ -74,7 +77,7 @@ By convention, this is arranged by starting each path with a unique prefix that belongs to you. For example, paths used internally at Google all begin with 'google', and paths denoting remote repositories begin with the path to the code, -such as 'code.google.com/p/project'. +such as 'github.com/user/repo'. As a special case, if the package list is a list of .go files from a single directory, the command is applied to a single synthesized @@ -192,7 +195,7 @@ example.org/repo or repo.git. When a version control system supports multiple protocols, each is tried in turn when downloading. For example, a Git -download tries git://, then https://, then http://. +download tries https://, then git+ssh://. If the import path is not a known code hosting site and also lacks a version control qualifier, the go tool attempts to fetch the import @@ -208,6 +211,10 @@ root. It must be a prefix or an exact match of the package being fetched with "go get". If it's not an exact match, another http request is made at the prefix to verify the <meta> tags match. +The meta tag should appear as early in the file as possible. +In particular, it should appear before any raw JavaScript or CSS, +to avoid confusing the go command's restricted parser. + The vcs is one of "git", "hg", "svn", etc, The repo-root is the root of the version control system @@ -217,10 +224,10 @@ For example, import "example.org/pkg/foo" -will result in the following request(s): +will result in the following requests: https://example.org/pkg/foo?go-get=1 (preferred) - http://example.org/pkg/foo?go-get=1 (fallback) + http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure) If that page contains the meta tag @@ -254,6 +261,11 @@ unless it is being referred to by that import path. In this way, import comments let package authors make sure the custom import path is used and not a direct path to the underlying code hosting site. +If the vendoring experiment is enabled (see 'go help gopath'), +then import path checking is disabled for code found within vendor trees. +This makes it possible to copy code into alternate locations in vendor trees +without needing to update import comments. + See https://golang.org/s/go14customimport for details. `, } @@ -275,10 +287,10 @@ standard Go tree. Each directory listed in GOPATH must have a prescribed structure: -The src/ directory holds source code. The path below 'src' +The src directory holds source code. The path below src determines the import path or executable name. -The pkg/ directory holds installed package objects. +The pkg directory holds installed package objects. As in the Go tree, each target operating system and architecture pair has its own subdirectory of pkg (pkg/GOOS_GOARCH). @@ -287,11 +299,11 @@ If DIR is a directory listed in the GOPATH, a package with source in DIR/src/foo/bar can be imported as "foo/bar" and has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". -The bin/ directory holds compiled commands. +The bin directory holds compiled commands. Each command is named for its source directory, but only the final element, not the entire path. That is, the command with source in DIR/src/foo/quux is installed into -DIR/bin/quux, not DIR/bin/foo/quux. The foo/ is stripped +DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped so that you can add DIR/bin to your PATH to get at the installed commands. If the GOBIN environment variable is set, commands are installed to the directory it names instead @@ -318,6 +330,168 @@ Here's an example directory layout: Go searches each directory listed in GOPATH to find source code, but new packages are always downloaded into the first directory in the list. + +See https://golang.org/doc/code.html for an example. + +Internal Directories + +Code in or below a directory named "internal" is importable only +by code in the directory tree rooted at the parent of "internal". +Here's an extended version of the directory layout above: + + /home/user/gocode/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + internal/ + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + + +The code in z.go is imported as "foo/internal/baz", but that +import statement can only appear in source files in the subtree +rooted at foo. The source files foo/f.go, foo/bar/x.go, and +foo/quux/y.go can all import "foo/internal/baz", but the source file +crash/bang/b.go cannot. + +See https://golang.org/s/go14internal for details. + +Vendor Directories + +Go 1.5 includes experimental support for using local copies +of external dependencies to satisfy imports of those dependencies, +often referred to as vendoring. Setting the environment variable +GO15VENDOREXPERIMENT=1 enables that experimental support. + +When the vendor experiment is enabled, +code below a directory named "vendor" is importable only +by code in the directory tree rooted at the parent of "vendor", +and only using an import path that omits the prefix up to and +including the vendor element. + +Here's the example from the previous section, +but with the "internal" directory renamed to "vendor" +and a new foo/vendor/crash/bang directory added: + + /home/user/gocode/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + vendor/ + crash/ + bang/ (go code in package bang) + b.go + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + +The same visibility rules apply as for internal, but the code +in z.go is imported as "baz", not as "foo/vendor/baz". + +Code in vendor directories deeper in the source tree shadows +code in higher directories. Within the subtree rooted at foo, an import +of "crash/bang" resolves to "foo/vendor/crash/bang", not the +top-level "crash/bang". + +Code in vendor directories is not subject to import path +checking (see 'go help importpath'). + +When the vendor experiment is enabled, 'go get' checks out +submodules when checking out or updating a git repository +(see 'go help get'). + +The vendoring semantics are an experiment, and they may change +in future releases. Once settled, they will be on by default. + +See https://golang.org/s/go15vendor for details. + `, +} + +var helpEnvironment = &Command{ + UsageLine: "environment", + Short: "environment variables", + Long: ` + +The go command, and the tools it invokes, examine a few different +environment variables. For many of these, you can see the default +value of on your system by running 'go env NAME', where NAME is the +name of the variable. + +General-purpose environment variables: + + GCCGO + The gccgo command to run for 'go build -compiler=gccgo'. + GOARCH + The architecture, or processor, for which to compile code. + Examples are amd64, 386, arm, ppc64. + GOBIN + The directory where 'go install' will install a command. + GOOS + The operating system for which to compile code. + Examples are linux, darwin, windows, netbsd. + GOPATH + See 'go help gopath'. + GORACE + Options for the race detector. + See https://golang.org/doc/articles/race_detector.html. + GOROOT + The root of the go tree. + +Environment variables for use with cgo: + + CC + The command to use to compile C code. + CGO_ENABLED + Whether the cgo command is supported. Either 0 or 1. + CGO_CFLAGS + Flags that cgo will pass to the compiler when compiling + C code. + CGO_CPPFLAGS + Flags that cgo will pass to the compiler when compiling + C or C++ code. + CGO_CXXFLAGS + Flags that cgo will pass to the compiler when compiling + C++ code. + CGO_LDFLAGS + Flags that cgo will pass to the compiler when linking. + CXX + The command to use to compile C++ code. + +Architecture-specific environment variables: + + GOARM + For GOARCH=arm, the ARM architecture for which to compile. + Valid values are 5, 6, 7. + GO386 + For GOARCH=386, the floating point instruction set. + Valid values are 387, sse2. + +Special-purpose environment variables: + + GOROOT_FINAL + The root of the installed Go tree, when it is + installed in a location other than where it is built. + File names in stack traces are rewritten from GOROOT to + GOROOT_FINAL. + GO15VENDOREXPERIMENT + Set to 1 to enable the Go 1.5 vendoring experiment. + GO_EXTLINK_ENABLED + Whether the linker should use external linking mode + when using -linkmode=auto with code that uses cgo. + Set to 0 to disable external linking mode, 1 to enable it. `, } @@ -333,10 +507,9 @@ the extension of the file name. These extensions are: Go source files. .c, .h C source files. - If the package uses cgo, these will be compiled with the - OS-native compiler (typically gcc); otherwise they will be - compiled with the Go-specific support compiler, - 5c, 6c, or 8c, etc. as appropriate. + If the package uses cgo or SWIG, these will be compiled with the + OS-native compiler (typically gcc); otherwise they will + trigger an error. .cc, .cpp, .cxx, .hh, .hpp, .hxx C++ source files. Only useful with cgo or SWIG, and always compiled with the OS-native compiler. @@ -345,10 +518,9 @@ the extension of the file name. These extensions are: compiled with the OS-native compiler. .s, .S Assembler source files. - If the package uses cgo, these will be assembled with the + If the package uses cgo or SWIG, these will be assembled with the OS-native assembler (typically gcc (sic)); otherwise they - will be assembled with the Go-specific support assembler, - 5a, 6a, or 8a, etc., as appropriate. + will be assembled with the Go assembler. .swig, .swigcxx SWIG definition files. .syso @@ -360,3 +532,43 @@ at the first item in the file that is not a blank line or //-style line comment. `, } + +var helpBuildmode = &Command{ + UsageLine: "buildmode", + Short: "description of build modes", + Long: ` +The 'go build' and 'go install' commands take a -buildmode argument which +indicates which kind of object file is to be built. Currently supported values +are: + + -buildmode=archive + Build the listed non-main packages into .a files. Packages named + main are ignored. + + -buildmode=c-archive + Build the listed main package, plus all packages it imports, + into a C archive file. The only callable symbols will be those + functions exported using a cgo //export comment. Requires + exactly one main package to be listed. + + -buildmode=c-shared + Build the listed main packages, plus all packages that they + import, into C shared libraries. The only callable symbols will + be those functions exported using a cgo //export comment. + Non-main packages are ignored. + + -buildmode=default + Listed main packages are built into executables and listed + non-main packages are built into .a files (the default + behavior). + + -buildmode=shared + Combine all the listed non-main packages into a single shared + library that will be used when building with the -linkshared + option. Packages named main are ignored. + + -buildmode=exe + Build the listed main packages and everything they import into + executables. Packages not named main are ignored. +`, +} diff --git a/libgo/go/cmd/go/http.go b/libgo/go/cmd/go/http.go index 107b820f28c..7979c41b11b 100644 --- a/libgo/go/cmd/go/http.go +++ b/libgo/go/cmd/go/http.go @@ -18,11 +18,25 @@ import ( "log" "net/http" "net/url" + "time" ) // httpClient is the default HTTP client, but a variable so it can be // changed by tests, without modifying http.DefaultClient. var httpClient = http.DefaultClient +var impatientHTTPClient = &http.Client{ + Timeout: time.Duration(5 * time.Second), +} + +type httpError struct { + status string + statusCode int + url string +} + +func (e *httpError) Error() string { + return fmt.Sprintf("%s: %s", e.url, e.status) +} // httpGET returns the data from an HTTP GET request for the given URL. func httpGET(url string) ([]byte, error) { @@ -32,7 +46,9 @@ func httpGET(url string) ([]byte, error) { } defer resp.Body.Close() if resp.StatusCode != 200 { - return nil, fmt.Errorf("%s: %s", url, resp.Status) + err := &httpError{status: resp.Status, statusCode: resp.StatusCode, url: url} + + return nil, err } b, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -43,7 +59,7 @@ func httpGET(url string) ([]byte, error) { // httpsOrHTTP returns the body of either the importPath's // https resource or, if unavailable, the http resource. -func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err error) { +func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body io.ReadCloser, err error) { fetch := func(scheme string) (urlStr string, res *http.Response, err error) { u, err := url.Parse(scheme + "://" + importPath) if err != nil { @@ -54,7 +70,11 @@ func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err erro if buildV { log.Printf("Fetching %s", urlStr) } - res, err = httpClient.Get(urlStr) + if security == insecure && scheme == "https" { // fail earlier + res, err = impatientHTTPClient.Get(urlStr) + } else { + res, err = httpClient.Get(urlStr) + } return } closeBody := func(res *http.Response) { @@ -72,7 +92,9 @@ func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err erro } } closeBody(res) - urlStr, res, err = fetch("http") + if security == insecure { + urlStr, res, err = fetch("http") + } } if err != nil { closeBody(res) diff --git a/libgo/go/cmd/go/list.go b/libgo/go/cmd/go/list.go index fbf96167feb..35c7cc4f2a7 100644 --- a/libgo/go/cmd/go/list.go +++ b/libgo/go/cmd/go/list.go @@ -21,9 +21,10 @@ List lists the packages named by the import paths, one per line. The default output shows the package import path: - code.google.com/p/google-api-go-client/books/v1 - code.google.com/p/goauth2/oauth - code.google.com/p/sqlite + bytes + encoding/json + github.com/gorilla/mux + golang.org/x/net/html The -f flag specifies an alternate format for the list, using the syntax of package template. The default output is equivalent to -f @@ -36,6 +37,7 @@ syntax of package template. The default output is equivalent to -f Name string // package name Doc string // package documentation string Target string // install path + Shlib string // the shared library that contains this package (only set when -linkshared) Goroot bool // is this package in the Go root? Standard bool // is this package part of the standard Go library? Stale bool // would 'go install' do anything for this package? @@ -126,6 +128,7 @@ var listJson = cmdList.Flag.Bool("json", false, "") var nl = []byte{'\n'} func runList(cmd *Command, args []string) { + buildModeInit() out := newTrackingWriter(os.Stdout) defer out.w.Flush() @@ -173,6 +176,10 @@ func runList(cmd *Command, args []string) { } for _, pkg := range load(args) { + // Show vendor-expanded paths in listing + pkg.TestImports = pkg.vendored(pkg.TestImports) + pkg.XTestImports = pkg.vendored(pkg.XTestImports) + do(pkg) } } diff --git a/libgo/go/cmd/go/main.go b/libgo/go/cmd/go/main.go index 9691f39c763..8ebde892599 100644 --- a/libgo/go/cmd/go/main.go +++ b/libgo/go/cmd/go/main.go @@ -5,6 +5,7 @@ package main import ( + "bufio" "bytes" "flag" "fmt" @@ -76,6 +77,7 @@ func (c *Command) Runnable() bool { var commands = []*Command{ cmdBuild, cmdClean, + cmdDoc, cmdEnv, cmdFix, cmdFmt, @@ -90,8 +92,10 @@ var commands = []*Command{ cmdVet, helpC, + helpBuildmode, helpFileType, helpGopath, + helpEnvironment, helpImportPath, helpPackages, helpTestflag, @@ -109,6 +113,8 @@ func setExitStatus(n int) { exitMu.Unlock() } +var origEnv []string + func main() { _ = go11tag flag.Usage = usage @@ -139,7 +145,7 @@ func main() { fmt.Fprintf(os.Stderr, "go: GOPATH entry cannot start with shell metacharacter '~': %q\n", p) os.Exit(2) } - if build.IsLocalImport(p) { + if !filepath.IsAbs(p) { fmt.Fprintf(os.Stderr, "go: GOPATH entry is relative; must be absolute path: %q.\nRun 'go help gopath' for usage.\n", p) os.Exit(2) } @@ -151,8 +157,20 @@ func main() { os.Exit(2) } + // Set environment (GOOS, GOARCH, etc) explicitly. + // In theory all the commands we invoke should have + // the same default computation of these as we do, + // but in practice there might be skew + // This makes sure we all agree. + origEnv = os.Environ() + for _, env := range mkEnv() { + if os.Getenv(env.name) != env.value { + os.Setenv(env.name, env.value) + } + } + for _, cmd := range commands { - if cmd.Name() == args[0] && cmd.Run != nil { + if cmd.Name() == args[0] && cmd.Runnable() { cmd.Flag.Usage = func() { cmd.Usage() } if cmd.CustomFlags { args = args[1:] @@ -179,13 +197,13 @@ Usage: The commands are: {{range .}}{{if .Runnable}} - {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} Use "go help [command]" for more information about a command. Additional help topics: {{range .}}{{if not .Runnable}} - {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} Use "go help [topic]" for more information about that topic. @@ -200,8 +218,8 @@ var documentationTemplate = `// Copyright 2011 The Go Authors. All rights reser // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// DO NOT EDIT THIS FILE. GENERATED BY mkdoc.sh. -// Edit the documentation in other files and rerun mkdoc.sh to generate this one. +// DO NOT EDIT THIS FILE. GENERATED BY mkalldocs.sh. +// Edit the documentation in other files and rerun mkalldocs.sh to generate this one. /* {{range .}}{{if .Short}}{{.Short | capitalize}} @@ -217,12 +235,35 @@ var documentationTemplate = `// Copyright 2011 The Go Authors. All rights reser package main ` +// An errWriter wraps a writer, recording whether a write error occurred. +type errWriter struct { + w io.Writer + err error +} + +func (w *errWriter) Write(b []byte) (int, error) { + n, err := w.w.Write(b) + if err != nil { + w.err = err + } + return n, err +} + // tmpl executes the given template text on data, writing the result to w. func tmpl(w io.Writer, text string, data interface{}) { t := template.New("top") t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) template.Must(t.Parse(text)) - if err := t.Execute(w, data); err != nil { + ew := &errWriter{w: w} + err := t.Execute(ew, data) + if ew.err != nil { + // I/O error writing. Ignore write on closed pipe. + if strings.Contains(ew.err.Error(), "pipe") { + os.Exit(1) + } + fatalf("writing output: %v", ew.err) + } + if err != nil { panic(err) } } @@ -236,13 +277,17 @@ func capitalize(s string) string { } func printUsage(w io.Writer) { - tmpl(w, usageTemplate, commands) + bw := bufio.NewWriter(w) + tmpl(bw, usageTemplate, commands) + bw.Flush() } func usage() { // special case "go test -h" if len(os.Args) > 1 && os.Args[1] == "test" { - help([]string{"testflag"}) + os.Stdout.WriteString(testUsage + "\n\n" + + strings.TrimSpace(testFlag1) + "\n\n" + + strings.TrimSpace(testFlag2) + "\n") os.Exit(2) } printUsage(os.Stderr) @@ -308,7 +353,7 @@ func importPathsNoDotExpansion(args []string) []string { } else { a = path.Clean(a) } - if a == "all" || a == "std" { + if a == "all" || a == "std" || a == "cmd" { out = append(out, allPackages(a)...) continue } @@ -401,11 +446,10 @@ func runOut(dir string, cmdargs ...interface{}) []byte { // The environment is the current process's environment // but with an updated $PWD, so that an os.Getwd in the // child will be faster. -func envForDir(dir string) []string { - env := os.Environ() +func envForDir(dir string, base []string) []string { // Internally we only use rooted paths, so dir is rooted. // Even if dir is not rooted, no harm done. - return mergeEnvLists([]string{"PWD=" + dir}, env) + return mergeEnvLists([]string{"PWD=" + dir}, base) } // mergeEnvLists merges the two environment lists such that @@ -458,6 +502,28 @@ func hasPathPrefix(s, prefix string) bool { } } +// hasFilePathPrefix reports whether the filesystem path s begins with the +// elements in prefix. +func hasFilePathPrefix(s, prefix string) bool { + sv := strings.ToUpper(filepath.VolumeName(s)) + pv := strings.ToUpper(filepath.VolumeName(prefix)) + s = s[len(sv):] + prefix = prefix[len(pv):] + switch { + default: + return false + case sv != pv: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == filepath.Separator { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix + } +} + // treeCanMatchPattern(pattern)(name) reports whether // name or children of name can possibly match pattern. // Pattern is the same limited glob accepted by matchPattern. @@ -475,8 +541,8 @@ func treeCanMatchPattern(pattern string) func(name string) bool { // allPackages returns all the packages that can be found // under the $GOPATH directories and $GOROOT matching pattern. -// The pattern is either "all" (all packages), "std" (standard packages) -// or a path including "...". +// The pattern is either "all" (all packages), "std" (standard packages), +// "cmd" (standard commands), or a path including "...". func allPackages(pattern string) []string { pkgs := matchPackages(pattern) if len(pkgs) == 0 { @@ -488,7 +554,7 @@ func allPackages(pattern string) []string { func matchPackages(pattern string) []string { match := func(string) bool { return true } treeCanMatch := func(string) bool { return true } - if pattern != "all" && pattern != "std" { + if pattern != "all" && pattern != "std" && pattern != "cmd" { match = matchPattern(pattern) treeCanMatch = treeCanMatchPattern(pattern) } @@ -501,47 +567,16 @@ func matchPackages(pattern string) []string { } var pkgs []string - // Commands - cmd := filepath.Join(goroot, "src/cmd") + string(filepath.Separator) - filepath.Walk(cmd, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() || path == cmd { - return nil - } - name := path[len(cmd):] - if !treeCanMatch(name) { - return filepath.SkipDir - } - // Commands are all in cmd/, not in subdirectories. - if strings.Contains(name, string(filepath.Separator)) { - return filepath.SkipDir - } - - // We use, e.g., cmd/gofmt as the pseudo import path for gofmt. - name = "cmd/" + name - if have[name] { - return nil - } - have[name] = true - if !match(name) { - return nil - } - _, err = buildContext.ImportDir(path, 0) - if err != nil { - if _, noGo := err.(*build.NoGoError); !noGo { - log.Print(err) - } - return nil - } - pkgs = append(pkgs, name) - return nil - }) - for _, src := range buildContext.SrcDirs() { - if pattern == "std" && src != gorootSrc { + if (pattern == "std" || pattern == "cmd") && src != gorootSrc { continue } src = filepath.Clean(src) + string(filepath.Separator) - filepath.Walk(src, func(path string, fi os.FileInfo, err error) error { + root := src + if pattern == "cmd" { + root += "cmd" + string(filepath.Separator) + } + filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil || !fi.IsDir() || path == src { return nil } @@ -553,7 +588,10 @@ func matchPackages(pattern string) []string { } name := filepath.ToSlash(path[len(src):]) - if pattern == "std" && strings.Contains(name, ".") { + if pattern == "std" && (strings.Contains(name, ".") || name == "cmd") { + // The name "std" is only the standard library. + // If the name has a dot, assume it's a domain name for go get, + // and if the name is cmd, it's the root of the command tree. return filepath.SkipDir } if !treeCanMatch(name) { @@ -659,7 +697,7 @@ func stringList(args ...interface{}) []string { case string: x = append(x, arg) default: - panic("stringList: invalid argument") + panic("stringList: invalid argument of type " + fmt.Sprintf("%T", arg)) } } return x diff --git a/libgo/go/cmd/go/note.go b/libgo/go/cmd/go/note.go new file mode 100644 index 00000000000..97e18651e4a --- /dev/null +++ b/libgo/go/cmd/go/note.go @@ -0,0 +1,116 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "debug/elf" + "encoding/binary" + "fmt" + "io" + "os" +) + +func readAligned4(r io.Reader, sz int32) ([]byte, error) { + full := (sz + 3) &^ 3 + data := make([]byte, full) + _, err := io.ReadFull(r, data) + if err != nil { + return nil, err + } + data = data[:sz] + return data, nil +} + +func readELFNote(filename, name string, typ int32) ([]byte, error) { + f, err := elf.Open(filename) + if err != nil { + return nil, err + } + for _, sect := range f.Sections { + if sect.Type != elf.SHT_NOTE { + continue + } + r := sect.Open() + for { + var namesize, descsize, noteType int32 + err = binary.Read(r, f.ByteOrder, &namesize) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("read namesize failed: %v", err) + } + err = binary.Read(r, f.ByteOrder, &descsize) + if err != nil { + return nil, fmt.Errorf("read descsize failed: %v", err) + } + err = binary.Read(r, f.ByteOrder, ¬eType) + if err != nil { + return nil, fmt.Errorf("read type failed: %v", err) + } + noteName, err := readAligned4(r, namesize) + if err != nil { + return nil, fmt.Errorf("read name failed: %v", err) + } + desc, err := readAligned4(r, descsize) + if err != nil { + return nil, fmt.Errorf("read desc failed: %v", err) + } + if name == string(noteName) && typ == noteType { + return desc, nil + } + } + } + return nil, nil +} + +var elfGoNote = []byte("Go\x00\x00") + +// readELFGoBuildID the Go build ID string from an ELF binary. +// The Go build ID is stored in a note described by an ELF PT_NOTE prog header. +// The caller has already opened filename, to get f, and read the first 4 kB out, in data. +func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { + // Assume the note content is in the first 4 kB, already read. + // Rewrite the ELF header to set shnum to 0, so that we can pass + // the data to elf.NewFile and it will decode the Prog list but not + // try to read the section headers and the string table from disk. + // That's a waste of I/O when all we care about is the Prog list + // and the one ELF note. + switch elf.Class(data[elf.EI_CLASS]) { + case elf.ELFCLASS32: + data[48] = 0 + data[49] = 0 + case elf.ELFCLASS64: + data[60] = 0 + data[61] = 0 + } + + const elfGoBuildIDTag = 4 + + ef, err := elf.NewFile(bytes.NewReader(data)) + if err != nil { + return "", &os.PathError{Path: filename, Op: "parse", Err: err} + } + for _, p := range ef.Progs { + if p.Type != elf.PT_NOTE || p.Off >= uint64(len(data)) || p.Off+p.Filesz >= uint64(len(data)) || p.Filesz < 16 { + continue + } + + note := data[p.Off : p.Off+p.Filesz] + nameSize := ef.ByteOrder.Uint32(note) + valSize := ef.ByteOrder.Uint32(note[4:]) + tag := ef.ByteOrder.Uint32(note[8:]) + name := note[12:16] + if nameSize != 4 || 16+valSize > uint32(len(note)) || tag != elfGoBuildIDTag || !bytes.Equal(name, elfGoNote) { + continue + } + + return string(note[16 : 16+valSize]), nil + } + + // No note. Treat as successful but build ID empty. + return "", nil +} diff --git a/libgo/go/cmd/go/note_test.go b/libgo/go/cmd/go/note_test.go new file mode 100644 index 00000000000..3d644518c68 --- /dev/null +++ b/libgo/go/cmd/go/note_test.go @@ -0,0 +1,49 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main_test + +import ( + main "cmd/go" + "runtime" + "testing" +) + +func TestNoteReading(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("hello.go", `package main; func main() { print("hello, world\n") }`) + const buildID = "TestNoteReading-Build-ID" + tg.run("build", "-ldflags", "-buildid="+buildID, "-o", tg.path("hello.exe"), tg.path("hello.go")) + id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe")) + if err != nil { + t.Fatalf("reading build ID from hello binary: %v", err) + } + if id != buildID { + t.Fatalf("buildID in hello binary = %q, want %q", id, buildID) + } + + if runtime.GOOS == "linux" && runtime.GOARCH == "ppc64le" { + t.Skipf("skipping - golang.org/issue/11184") + } + + switch runtime.GOOS { + case "plan9": + // no external linking + t.Logf("no external linking - skipping linkmode=external test") + + case "solaris": + t.Logf("skipping - golang.org/issue/12178") + + default: + tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go")) + id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe")) + if err != nil { + t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err) + } + if id != buildID { + t.Fatalf("buildID in hello binary = %q, want %q (linkmode=external)", id, buildID) + } + } +} diff --git a/libgo/go/cmd/go/pkg.go b/libgo/go/cmd/go/pkg.go index ef440dd3b74..ff5236e90ab 100644 --- a/libgo/go/cmd/go/pkg.go +++ b/libgo/go/cmd/go/pkg.go @@ -6,18 +6,21 @@ package main import ( "bytes" + "crypto/sha1" "errors" "fmt" "go/build" "go/scanner" "go/token" + "io" + "io/ioutil" "os" pathpkg "path" "path/filepath" "runtime" "sort" + "strconv" "strings" - "time" "unicode" ) @@ -32,6 +35,7 @@ type Package struct { Name string `json:",omitempty"` // package name Doc string `json:",omitempty"` // package documentation string Target string `json:",omitempty"` // install path + Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) Goroot bool `json:",omitempty"` // is this package found in the Go root? Standard bool `json:",omitempty"` // is this package part of the standard Go library? Stale bool `json:",omitempty"` // would 'go install' do anything for this package? @@ -93,6 +97,35 @@ type Package struct { coverMode string // preprocess Go source files with the coverage tool in this mode coverVars map[string]*CoverVar // variables created by coverage analysis omitDWARF bool // tell linker not to write DWARF information + buildID string // expected build ID for generated package + gobinSubdir bool // install target would be subdir of GOBIN +} + +// vendored returns the vendor-resolved version of imports, +// which should be p.TestImports or p.XTestImports, NOT p.Imports. +// The imports in p.TestImports and p.XTestImports are not recursively +// loaded during the initial load of p, so they list the imports found in +// the source file, but most processing should be over the vendor-resolved +// import paths. We do this resolution lazily both to avoid file system work +// and because the eventual real load of the test imports (during 'go test') +// can produce better error messages if it starts with the original paths. +// The initial load of p loads all the non-test imports and rewrites +// the vendored paths, so nothing should ever call p.vendored(p.Imports). +func (p *Package) vendored(imports []string) []string { + if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { + panic("internal error: p.vendored(p.Imports) called") + } + seen := make(map[string]bool) + var all []string + for _, path := range imports { + path, _ = vendoredImportPath(p, path) + if !seen[path] { + seen[path] = true + all = append(all, path) + } + } + sort.Strings(all) + return all } // CoverVar holds the name of the generated coverage variables targeting the named file. @@ -104,6 +137,13 @@ type CoverVar struct { func (p *Package) copyBuild(pp *build.Package) { p.build = pp + if pp.PkgTargetRoot != "" && buildPkgdir != "" { + old := pp.PkgTargetRoot + pp.PkgRoot = buildPkgdir + pp.PkgTargetRoot = buildPkgdir + pp.PkgObj = filepath.Join(buildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) + } + p.Dir = pp.Dir p.ImportPath = pp.ImportPath p.ImportComment = pp.ImportComment @@ -181,7 +221,7 @@ func (s *importStack) copy() []string { return append([]string{}, *s...) } -// shorterThan returns true if sp is shorter than t. +// shorterThan reports whether sp is shorter than t. // We use this to record the shortest import sequence // that leads to a particular package. func (sp *importStack) shorterThan(t []string) bool { @@ -214,6 +254,12 @@ func reloadPackage(arg string, stk *importStack) *Package { return loadPackage(arg, stk) } +// The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1. +// The variable is obnoxiously long so that years from now when people find it in +// their profiles and wonder what it does, there is some chance that a web search +// might answer the question. +var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1" + // dirToImportPath returns the pseudo-import path we use for a package // outside the Go path. It begins with _/ and then contains the full path // to the directory. If the package lives in c:\home\gopher\my\pkg then @@ -234,11 +280,29 @@ func makeImportValid(r rune) rune { return r } +// Mode flags for loadImport and download (in get.go). +const ( + // useVendor means that loadImport should do vendor expansion + // (provided the vendoring experiment is enabled). + // That is, useVendor means that the import path came from + // a source file and has not been vendor-expanded yet. + // Every import path should be loaded initially with useVendor, + // and then the expanded version (with the /vendor/ in it) gets + // recorded as the canonical import path. At that point, future loads + // of that package must not pass useVendor, because + // disallowVendor will reject direct use of paths containing /vendor/. + useVendor = 1 << iota + + // getTestDeps is for download (part of "go get") and indicates + // that test dependencies should be fetched too. + getTestDeps +) + // loadImport scans the directory named by path, which must be an import path, // but possibly a local import path (an absolute file system path or one beginning // with ./ or ../). A local relative path is interpreted relative to srcDir. // It returns a *Package describing the package found in that directory. -func loadImport(path string, srcDir string, stk *importStack, importPos []token.Position) *Package { +func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position, mode int) *Package { stk.push(path) defer stk.pop() @@ -246,15 +310,27 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token. // For a local import the identifier is the pseudo-import path // we create from the full directory to the package. // Otherwise it is the usual import path. + // For vendored imports, it is the expanded form. importPath := path + origPath := path isLocal := build.IsLocalImport(path) + var vendorSearch []string if isLocal { importPath = dirToImportPath(filepath.Join(srcDir, path)) + } else if mode&useVendor != 0 { + path, vendorSearch = vendoredImportPath(parent, path) + importPath = path } + if p := packageCache[importPath]; p != nil { if perr := disallowInternal(srcDir, p, stk); perr != p { return perr } + if mode&useVendor != 0 { + if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { + return perr + } + } return reusePackage(p, stk) } @@ -270,11 +346,34 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token. // TODO: After Go 1, decide when to pass build.AllowBinary here. // See issue 3268 for mistakes to avoid. bp, err := buildContext.Import(path, srcDir, build.ImportComment) + + // If we got an error from go/build about package not found, + // it contains the directories from $GOROOT and $GOPATH that + // were searched. Add to that message the vendor directories + // that were searched. + if err != nil && len(vendorSearch) > 0 { + // NOTE(rsc): The direct text manipulation here is fairly awful, + // but it avoids defining new go/build API (an exported error type) + // late in the Go 1.5 release cycle. If this turns out to be a more general + // problem we could define a real error type when the decision can be + // considered more carefully. + text := err.Error() + if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") { + old := strings.SplitAfter(text, "\n") + lines := []string{old[0]} + for _, dir := range vendorSearch { + lines = append(lines, "\t"+dir+" (vendor tree)\n") + } + lines = append(lines, old[1:]...) + err = errors.New(strings.Join(lines, "")) + } + } bp.ImportPath = importPath if gobin != "" { bp.BinDir = gobin } - if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path { + if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && + (!go15VendorExperiment || (!strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/"))) { err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) } p.load(stk, bp, err) @@ -287,10 +386,83 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token. if perr := disallowInternal(srcDir, p, stk); perr != p { return perr } + if mode&useVendor != 0 { + if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { + return perr + } + } return p } +var isDirCache = map[string]bool{} + +func isDir(path string) bool { + result, ok := isDirCache[path] + if ok { + return result + } + + fi, err := os.Stat(path) + result = err == nil && fi.IsDir() + isDirCache[path] = result + return result +} + +// vendoredImportPath returns the expansion of path when it appears in parent. +// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, +// x/vendor/path, vendor/path, or else stay x/y/z if none of those exist. +// vendoredImportPath returns the expanded path or, if no expansion is found, the original. +// If no expansion is found, vendoredImportPath also returns a list of vendor directories +// it searched along the way, to help prepare a useful error message should path turn +// out not to exist. +func vendoredImportPath(parent *Package, path string) (found string, searched []string) { + if parent == nil || parent.Root == "" || !go15VendorExperiment { + return path, nil + } + dir := filepath.Clean(parent.Dir) + root := filepath.Join(parent.Root, "src") + if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator { + fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator)) + } + vpath := "vendor/" + path + for i := len(dir); i >= len(root); i-- { + if i < len(dir) && dir[i] != filepath.Separator { + continue + } + // Note: checking for the vendor directory before checking + // for the vendor/path directory helps us hit the + // isDir cache more often. It also helps us prepare a more useful + // list of places we looked, to report when an import is not found. + if !isDir(filepath.Join(dir[:i], "vendor")) { + continue + } + targ := filepath.Join(dir[:i], vpath) + if isDir(targ) { + // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. + // We know the import path for parent's dir. + // We chopped off some number of path elements and + // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. + // Now we want to know the import path for that directory. + // Construct it by chopping the same number of path elements + // (actually the same number of bytes) from parent's import path + // and then append /vendor/path. + chopped := len(dir) - i + if chopped == len(parent.ImportPath)+1 { + // We walked up from c:\gopath\src\foo\bar + // and found c:\gopath\src\vendor\path. + // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). + // Use "vendor/path" without any prefix. + return vpath, nil + } + return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil + } + // Note the existence of a vendor directory in case path is not found anywhere. + searched = append(searched, targ) + } + return path, searched +} + // reusePackage reuses package p to satisfy the import at the top // of the import stack stk. If this use causes an import loop, // reusePackage updates p's error information to record the loop. @@ -324,11 +496,9 @@ func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { // An import of a path containing the element “internal” // is disallowed if the importing code is outside the tree // rooted at the parent of the “internal” directory. - // - // ... For Go 1.4, we will implement the rule first for $GOROOT, but not $GOPATH. - // Only applies to $GOROOT. - if !p.Standard { + // There was an error loading the package; stop here. + if p.Error != nil { return p } @@ -385,6 +555,105 @@ func findInternal(path string) (index int, ok bool) { return 0, false } +// disallowVendor checks that srcDir is allowed to import p as path. +// If the import is allowed, disallowVendor returns the original package p. +// If not, it returns a new package containing just an appropriate error. +func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package { + if !go15VendorExperiment { + return p + } + + // The stack includes p.ImportPath. + // If that's the only thing on the stack, we started + // with a name given on the command line, not an + // import. Anything listed on the command line is fine. + if len(*stk) == 1 { + return p + } + + if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { + return perr + } + + // Paths like x/vendor/y must be imported as y, never as x/vendor/y. + if i, ok := findVendor(path); ok { + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.copy(), + Err: "must be imported as " + path[i+len("vendor/"):], + } + perr.Incomplete = true + return &perr + } + + return p +} + +// disallowVendorVisibility checks that srcDir is allowed to import p. +// The rules are the same as for /internal/ except that a path ending in /vendor +// is not subject to the rules, only subdirectories of vendor. +// This allows people to have packages and commands named vendor, +// for maximal compatibility with existing source trees. +func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package { + // The stack includes p.ImportPath. + // If that's the only thing on the stack, we started + // with a name given on the command line, not an + // import. Anything listed on the command line is fine. + if len(*stk) == 1 { + return p + } + + // Check for "vendor" element. + i, ok := findVendor(p.ImportPath) + if !ok { + return p + } + + // Vendor is present. + // Map import path back to directory corresponding to parent of vendor. + if i > 0 { + i-- // rewind over slash in ".../vendor" + } + truncateTo := i + len(p.Dir) - len(p.ImportPath) + if truncateTo < 0 || len(p.Dir) < truncateTo { + return p + } + parent := p.Dir[:truncateTo] + if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { + return p + } + + // Vendor is present, and srcDir is outside parent's tree. Not allowed. + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.copy(), + Err: "use of vendored package not allowed", + } + perr.Incomplete = true + return &perr +} + +// findVendor looks for the last non-terminating "vendor" path element in the given import path. +// If there isn't one, findVendor returns ok=false. +// Otherwise, findInternal returns ok=true and the index of the "vendor". +// +// Note that terminating "vendor" elements don't count: "x/vendor" is its own package, +// not the vendored copy of an import "" (the empty import path). +// This will allow people to have packages or commands named vendor. +// This may help reduce breakage, or it may just be confusing. We'll see. +func findVendor(path string) (index int, ok bool) { + // Two cases, depending on internal at start of string or not. + // The order matters: we must return the index of the final element, + // because the final one is where the effective import path starts. + switch { + case strings.Contains(path, "/vendor/"): + return strings.LastIndex(path, "/vendor/") + 1, true + case strings.HasPrefix(path, "vendor/"): + return 0, true + } + return 0, false +} + type targetDir int const ( @@ -398,17 +667,23 @@ const ( var goTools = map[string]targetDir{ "cmd/addr2line": toTool, "cmd/api": toTool, + "cmd/asm": toTool, + "cmd/compile": toTool, "cmd/cgo": toTool, + "cmd/cover": toTool, + "cmd/dist": toTool, + "cmd/doc": toTool, "cmd/fix": toTool, "cmd/link": toTool, + "cmd/newlink": toTool, "cmd/nm": toTool, "cmd/objdump": toTool, "cmd/pack": toTool, "cmd/pprof": toTool, + "cmd/trace": toTool, + "cmd/vet": toTool, "cmd/yacc": toTool, - "golang.org/x/tools/cmd/cover": toTool, "golang.org/x/tools/cmd/godoc": toBin, - "golang.org/x/tools/cmd/vet": toTool, "code.google.com/p/go.tools/cmd/cover": stalePath, "code.google.com/p/go.tools/cmd/godoc": stalePath, "code.google.com/p/go.tools/cmd/vet": stalePath, @@ -471,7 +746,15 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package return p } - if p.Name == "main" { + useBindir := p.Name == "main" + if !p.Standard { + switch buildBuildmode { + case "c-archive", "c-shared": + useBindir = false + } + } + + if useBindir { // Report an error when the old code.google.com/p/go.tools paths are used. if goTools[p.ImportPath] == stalePath { newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) @@ -493,6 +776,11 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } else if p.build.BinDir != "" { // Install to GOBIN or bin of GOPATH entry. p.target = filepath.Join(p.build.BinDir, elem) + if !p.Goroot && strings.Contains(elem, "/") && gobin != "" { + // Do not create $GOBIN/goos_goarch/elem. + p.target = "" + p.gobinSubdir = true + } } if goTools[p.ImportPath] == toTool { // This is for 'go tool'. @@ -508,6 +796,21 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package p.target = "" } else { p.target = p.build.PkgObj + if buildLinkshared { + shlibnamefile := p.target[:len(p.target)-2] + ".shlibname" + shlib, err := ioutil.ReadFile(shlibnamefile) + if err == nil { + libname := strings.TrimSpace(string(shlib)) + if buildContext.Compiler == "gccgo" { + p.Shlib = filepath.Join(p.build.PkgTargetRoot, "shlibs", libname) + } else { + p.Shlib = filepath.Join(p.build.PkgTargetRoot, libname) + + } + } else if !os.IsNotExist(err) { + fatalf("unexpected error reading %s: %v", shlibnamefile, err) + } + } } importPaths := p.Imports @@ -521,6 +824,14 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { importPaths = append(importPaths, "syscall") } + + // Currently build mode c-shared, or -linkshared, forces + // external linking mode, and external linking mode forces an + // import of runtime/cgo. + if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildLinkshared) { + importPaths = append(importPaths, "runtime/cgo") + } + // Everything depends on runtime, except runtime and unsafe. if !p.Standard || (p.ImportPath != "runtime" && p.ImportPath != "unsafe") { importPaths = append(importPaths, "runtime") @@ -529,6 +840,10 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { importPaths = append(importPaths, "runtime/race") } + // On ARM with GOARM=5, everything depends on math for the link. + if p.Name == "main" && goarch == "arm" { + importPaths = append(importPaths, "math") + } } // Build list of full paths to all Go files in the package, @@ -586,10 +901,20 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package if path == "C" { continue } - p1 := loadImport(path, p.Dir, stk, p.build.ImportPos[path]) + p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path], useVendor) if !reqStdPkgSrc && p1.Standard { continue } + if p1.Name == "main" { + p.Error = &PackageError{ + ImportStack: stk.copy(), + Err: fmt.Sprintf("import %q is a program, not an importable package", path), + } + pos := p.build.ImportPos[path] + if len(pos) > 0 { + p.Error.Pos = pos[0].String() + } + } if p1.local { if !p.local && p.Error == nil { p.Error = &PackageError{ @@ -601,13 +926,21 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package p.Error.Pos = pos[0].String() } } - path = p1.ImportPath - importPaths[i] = path + } + path = p1.ImportPath + importPaths[i] = path + if i < len(p.Imports) { + p.Imports[i] = path } deps[path] = p1 imports = append(imports, p1) for _, dep := range p1.deps { - deps[dep.ImportPath] = dep + // The same import path could produce an error or not, + // depending on what tries to import it. + // Prefer to record entries with errors, so we can report them. + if deps[dep.ImportPath] == nil || dep.Error != nil { + deps[dep.ImportPath] = dep + } } if p1.Incomplete { p.Incomplete = true @@ -637,12 +970,11 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } p.Target = p.target - // Check for C code compiled with Plan 9 C compiler. - // No longer allowed except in runtime and runtime/cgo, for now. - if len(p.CFiles) > 0 && !p.usesCgo() && (!p.Standard || p.ImportPath != "runtime") { + // The gc toolchain only permits C source files with cgo. + if len(p.CFiles) > 0 && !p.usesCgo() && !p.usesSwig() && buildContext.Compiler == "gc" { p.Error = &PackageError{ ImportStack: stk.copy(), - Err: fmt.Sprintf("C source files not allowed when not using cgo: %s", strings.Join(p.CFiles, " ")), + Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), } return p } @@ -660,6 +992,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } } + computeBuildID(p) return p } @@ -698,13 +1031,8 @@ func packageList(roots []*Package) []*Package { // computeStale computes the Stale flag in the package dag that starts // at the named pkgs (command-line arguments). func computeStale(pkgs ...*Package) { - topRoot := map[string]bool{} - for _, p := range pkgs { - topRoot[p.Root] = true - } - for _, p := range packageList(pkgs) { - p.Stale = isStale(p, topRoot) + p.Stale = isStale(p) } } @@ -714,8 +1042,269 @@ func computeStale(pkgs ...*Package) { // inspecting the version. var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") +// isStale and computeBuildID +// +// Theory of Operation +// +// There is an installed copy of the package (or binary). +// Can we reuse the installed copy, or do we need to build a new one? +// +// We can use the installed copy if it matches what we'd get +// by building a new one. The hard part is predicting that without +// actually running a build. +// +// To start, we must know the set of inputs to the build process that can +// affect the generated output. At a minimum, that includes the source +// files for the package and also any compiled packages imported by those +// source files. The *Package has these, and we use them. One might also +// argue for including in the input set: the build tags, whether the race +// detector is in use, the target operating system and architecture, the +// compiler and linker binaries being used, the additional flags being +// passed to those, the cgo binary being used, the additional flags cgo +// passes to the host C compiler, the host C compiler being used, the set +// of host C include files and installed C libraries, and so on. +// We include some but not all of this information. +// +// Once we have decided on a set of inputs, we must next decide how to +// tell whether the content of that set has changed since the last build +// of p. If there have been no changes, then we assume a new build would +// produce the same result and reuse the installed package or binary. +// But if there have been changes, then we assume a new build might not +// produce the same result, so we rebuild. +// +// There are two common ways to decide whether the content of the set has +// changed: modification times and content hashes. We use a mixture of both. +// +// The use of modification times (mtimes) was pioneered by make: +// assuming that a file's mtime is an accurate record of when that file was last written, +// and assuming that the modification time of an installed package or +// binary is the time that it was built, if the mtimes of the inputs +// predate the mtime of the installed object, then the build of that +// object saw those versions of the files, and therefore a rebuild using +// those same versions would produce the same object. In contrast, if any +// mtime of an input is newer than the mtime of the installed object, a +// change has occurred since the build, and the build should be redone. +// +// Modification times are attractive because the logic is easy to +// understand and the file system maintains the mtimes automatically +// (less work for us). Unfortunately, there are a variety of ways in +// which the mtime approach fails to detect a change and reuses a stale +// object file incorrectly. (Making the opposite mistake, rebuilding +// unnecessarily, is only a performance problem and not a correctness +// problem, so we ignore that one.) +// +// As a warmup, one problem is that to be perfectly precise, we need to +// compare the input mtimes against the time at the beginning of the +// build, but the object file time is the time at the end of the build. +// If an input file changes after being read but before the object is +// written, the next build will see an object newer than the input and +// will incorrectly decide that the object is up to date. We make no +// attempt to detect or solve this problem. +// +// Another problem is that due to file system imprecision, an input and +// output that are actually ordered in time have the same mtime. +// This typically happens on file systems with 1-second (or, worse, +// 2-second) mtime granularity and with automated scripts that write an +// input and then immediately run a build, or vice versa. If an input and +// an output have the same mtime, the conservative behavior is to treat +// the output as out-of-date and rebuild. This can cause one or more +// spurious rebuilds, but only for 1 second, until the object finally has +// an mtime later than the input. +// +// Another problem is that binary distributions often set the mtime on +// all files to the same time. If the distribution includes both inputs +// and cached build outputs, the conservative solution to the previous +// problem will cause unnecessary rebuilds. Worse, in such a binary +// distribution, those rebuilds might not even have permission to update +// the cached build output. To avoid these write errors, if an input and +// output have the same mtime, we assume the output is up-to-date. +// This is the opposite of what the previous problem would have us do, +// but binary distributions are more common than instances of the +// previous problem. +// +// A variant of the last problem is that some binary distributions do not +// set the mtime on all files to the same time. Instead they let the file +// system record mtimes as the distribution is unpacked. If the outputs +// are unpacked before the inputs, they'll be older and a build will try +// to rebuild them. That rebuild might hit the same write errors as in +// the last scenario. We don't make any attempt to solve this, and we +// haven't had many reports of it. Perhaps the only time this happens is +// when people manually unpack the distribution, and most of the time +// that's done as the same user who will be using it, so an initial +// rebuild on first use succeeds quietly. +// +// More generally, people and programs change mtimes on files. The last +// few problems were specific examples of this, but it's a general problem. +// For example, instead of a binary distribution, copying a home +// directory from one directory or machine to another might copy files +// but not preserve mtimes. If the inputs are new than the outputs on the +// first machine but copied first, they end up older than the outputs on +// the second machine. +// +// Because many other build systems have the same sensitivity to mtimes, +// most programs manipulating source code take pains not to break the +// mtime assumptions. For example, Git does not set the mtime of files +// during a checkout operation, even when checking out an old version of +// the code. This decision was made specifically to work well with +// mtime-based build systems. +// +// The killer problem, though, for mtime-based build systems is that the +// build only has access to the mtimes of the inputs that still exist. +// If it is possible to remove an input without changing any other inputs, +// a later build will think the object is up-to-date when it is not. +// This happens for Go because a package is made up of all source +// files in a directory. If a source file is removed, there is no newer +// mtime available recording that fact. The mtime on the directory could +// be used, but it also changes when unrelated files are added to or +// removed from the directory, so including the directory mtime would +// cause unnecessary rebuilds, possibly many. It would also exacerbate +// the problems mentioned earlier, since even programs that are careful +// to maintain mtimes on files rarely maintain mtimes on directories. +// +// A variant of the last problem is when the inputs change for other +// reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg +// into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. +// If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must +// rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. +// If Go 1.5 has just been installed, perhaps the compiler will have a +// newer mtime; since the compiler is considered an input, that would +// trigger a rebuild. But only once, and only the last Go 1.4 build of +// mypkg.a happened before Go 1.5 was installed. If a user has the two +// versions installed in different locations and flips back and forth, +// mtimes alone cannot tell what to do. Changing the toolchain is +// changing the set of inputs, without affecting any mtimes. +// +// To detect the set of inputs changing, we turn away from mtimes and to +// an explicit data comparison. Specifically, we build a list of the +// inputs to the build, compute its SHA1 hash, and record that as the +// ``build ID'' in the generated object. At the next build, we can +// recompute the buid ID and compare it to the one in the generated +// object. If they differ, the list of inputs has changed, so the object +// is out of date and must be rebuilt. +// +// Because this build ID is computed before the build begins, the +// comparison does not have the race that mtime comparison does. +// +// Making the build sensitive to changes in other state is +// straightforward: include the state in the build ID hash, and if it +// changes, so does the build ID, triggering a rebuild. +// +// To detect changes in toolchain, we include the toolchain version in +// the build ID hash for package runtime, and then we include the build +// IDs of all imported packages in the build ID for p. +// +// It is natural to think about including build tags in the build ID, but +// the naive approach of just dumping the tags into the hash would cause +// spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' +// produce the same binaries (assuming neverusedtag is never used). +// A more precise approach would be to include only tags that have an +// effect on the build. But the effect of a tag on the build is to +// include or exclude a file from the compilation, and that file list is +// already in the build ID hash. So the build ID is already tag-sensitive +// in a perfectly precise way. So we do NOT explicitly add build tags to +// the build ID hash. +// +// We do not include as part of the build ID the operating system, +// architecture, or whether the race detector is enabled, even though all +// three have an effect on the output, because that information is used +// to decide the install location. Binaries for linux and binaries for +// darwin are written to different directory trees; including that +// information in the build ID is unnecessary (although it would be +// harmless). +// +// TODO(rsc): Investigate the cost of putting source file content into +// the build ID hash as a replacement for the use of mtimes. Using the +// file content would avoid all the mtime problems, but it does require +// reading all the source files, something we avoid today (we read the +// beginning to find the build tags and the imports, but we stop as soon +// as we see the import block is over). If the package is stale, the compiler +// is going to read the files anyway. But if the package is up-to-date, the +// read is overhead. +// +// TODO(rsc): Investigate the complexity of making the build more +// precise about when individual results are needed. To be fully precise, +// there are two results of a compilation: the entire .a file used by the link +// and the subpiece used by later compilations (__.PKGDEF only). +// If a rebuild is needed but produces the previous __.PKGDEF, then +// no more recompilation due to the rebuilt package is needed, only +// relinking. To date, there is nothing in the Go command to express this. +// +// Special Cases +// +// When the go command makes the wrong build decision and does not +// rebuild something it should, users fall back to adding the -a flag. +// Any common use of the -a flag should be considered prima facie evidence +// that isStale is returning an incorrect false result in some important case. +// Bugs reported in the behavior of -a itself should prompt the question +// ``Why is -a being used at all? What bug does that indicate?'' +// +// There is a long history of changes to isStale to try to make -a into a +// suitable workaround for bugs in the mtime-based decisions. +// It is worth recording that history to inform (and, as much as possible, deter) future changes. +// +// (1) Before the build IDs were introduced, building with alternate tags +// would happily reuse installed objects built without those tags. +// For example, "go build -tags netgo myprog.go" would use the installed +// copy of package net, even if that copy had been built without netgo. +// (The netgo tag controls whether package net uses cgo or pure Go for +// functionality such as name resolution.) +// Using the installed non-netgo package defeats the purpose. +// +// Users worked around this with "go build -tags netgo -a myprog.go". +// +// Build IDs have made that workaround unnecessary: +// "go build -tags netgo myprog.go" +// cannot use a non-netgo copy of package net. +// +// (2) Before the build IDs were introduced, building with different toolchains, +// especially changing between toolchains, tried to reuse objects stored in +// $GOPATH/pkg, resulting in link-time errors about object file mismatches. +// +// Users worked around this with "go install -a ./...". +// +// Build IDs have made that workaround unnecessary: +// "go install ./..." will rebuild any objects it finds that were built against +// a different toolchain. +// +// (3) The common use of "go install -a ./..." led to reports of problems +// when the -a forced the rebuild of the standard library, which for some +// users was not writable. Because we didn't understand that the real +// problem was the bug -a was working around, we changed -a not to +// apply to the standard library. +// +// (4) The common use of "go build -tags netgo -a myprog.go" broke +// when we changed -a not to apply to the standard library, because +// if go build doesn't rebuild package net, it uses the non-netgo version. +// +// Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". +// The -installsuffix here is making the go command look for packages +// in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. +// Since the former presumably doesn't exist, go build decides to rebuild +// everything, including the standard library. Since go build doesn't +// install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, +// so repeated invocations continue to work. +// +// If the use of -a wasn't a red flag, the use of -installsuffix to point to +// a non-existent directory in a command that installs nothing should +// have been. +// +// (5) Now that (1) and (2) no longer need -a, we have removed the kludge +// introduced in (3): once again, -a means ``rebuild everything,'' not +// ``rebuild everything except the standard library.'' Only Go 1.4 had +// the restricted meaning. +// +// In addition to these cases trying to trigger rebuilds, there are +// special cases trying NOT to trigger rebuilds. The main one is that for +// a variety of reasons (see above), the install process for a Go release +// cannot be relied upon to set the mtimes such that the go command will +// think the standard library is up to date. So the mtime evidence is +// ignored for the standard library if we find ourselves in a release +// version of Go. Build ID-based staleness checks still apply to the +// standard library, even in release versions. This makes +// 'go build -tags netgo' work, among other things. + // isStale reports whether package p needs to be rebuilt. -func isStale(p *Package, topRoot map[string]bool) bool { +func isStale(p *Package) bool { if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { // fake, builtin package return false @@ -734,28 +1323,68 @@ func isStale(p *Package, topRoot map[string]bool) bool { return false } - // If we are running a release copy of Go, do not rebuild the standard packages. - // They may not be writable anyway, but they are certainly not changing. - // This makes 'go build -a' skip the standard packages when using an official release. - // See issue 4106 and issue 8290. - pkgBuildA := buildA - if p.Standard && isGoRelease { - pkgBuildA = false + // If the -a flag is given, rebuild everything. + if buildA { + return true } - if pkgBuildA || p.target == "" || p.Stale { + // If there's no install target or it's already marked stale, we have to rebuild. + if p.target == "" || p.Stale { return true } // Package is stale if completely unbuilt. - var built time.Time - if fi, err := os.Stat(p.target); err == nil { - built = fi.ModTime() + fi, err := os.Stat(p.target) + if err != nil { + return true } - if built.IsZero() { + + // Package is stale if the expected build ID differs from the + // recorded build ID. This catches changes like a source file + // being removed from a package directory. See issue 3895. + // It also catches changes in build tags that affect the set of + // files being compiled. See issue 9369. + // It also catches changes in toolchain, like when flipping between + // two versions of Go compiling a single GOPATH. + // See issue 8290 and issue 10702. + targetBuildID, err := readBuildID(p) + if err == nil && targetBuildID != p.buildID { return true } + // Package is stale if a dependency is. + for _, p1 := range p.deps { + if p1.Stale { + return true + } + } + + // The checks above are content-based staleness. + // We assume they are always accurate. + // + // The checks below are mtime-based staleness. + // We hope they are accurate, but we know that they fail in the case of + // prebuilt Go installations that don't preserve the build mtimes + // (for example, if the pkg/ mtimes are before the src/ mtimes). + // See the large comment above isStale for details. + + // If we are running a release copy of Go and didn't find a content-based + // reason to rebuild the standard packages, do not rebuild them. + // They may not be writable anyway, but they are certainly not changing. + // This makes 'go build' skip the standard packages when + // using an official release, even when the mtimes have been changed. + // See issue 3036, issue 3149, issue 4106, issue 8290. + // (If a change to a release tree must be made by hand, the way to force the + // install is to run make.bash, which will remove the old package archives + // before rebuilding.) + if p.Standard && isGoRelease { + return false + } + + // Time-based staleness. + + built := fi.ModTime() + olderThan := func(file string) bool { fi, err := os.Stat(file) return err != nil || fi.ModTime().After(built) @@ -763,7 +1392,7 @@ func isStale(p *Package, topRoot map[string]bool) bool { // Package is stale if a dependency is, or if a dependency is newer. for _, p1 := range p.deps { - if p1.Stale || p1.target != "" && olderThan(p1.target) { + if p1.target != "" && olderThan(p1.target) { return true } } @@ -775,8 +1404,12 @@ func isStale(p *Package, topRoot map[string]bool) bool { // back-dated, as some binary distributions may do, but it does handle // a very common case. // See issue 3036. - // Assume code in $GOROOT is up to date, since it may not be writeable. - // See issue 4106. + // Exclude $GOROOT, under the assumption that people working on + // the compiler may want to control when everything gets rebuilt, + // and people updating the Go repository will run make.bash or all.bash + // and get a full rebuild anyway. + // Excluding $GOROOT used to also fix issue 4106, but that's now + // taken care of above (at least when the installed Go is a released version). if p.Root != goroot { if olderThan(buildToolchain.compiler()) { return true @@ -786,19 +1419,43 @@ func isStale(p *Package, topRoot map[string]bool) bool { } } - // Have installed copy, probably built using current compilers, - // and built after its imported packages. The only reason now - // that we'd have to rebuild it is if the sources were newer than - // the package. If a package p is not in the same tree as any - // package named on the command-line, assume it is up-to-date - // no matter what the modification times on the source files indicate. - // This avoids rebuilding $GOROOT packages when people are - // working outside the Go root, and it effectively makes each tree - // listed in $GOPATH a separate compilation world. - // See issue 3149. - if p.Root != "" && !topRoot[p.Root] { - return false - } + // Note: Until Go 1.5, we had an additional shortcut here. + // We built a list of the workspace roots ($GOROOT, each $GOPATH) + // containing targets directly named on the command line, + // and if p were not in any of those, it would be treated as up-to-date + // as long as it is built. The goal was to avoid rebuilding a system-installed + // $GOROOT, unless something from $GOROOT were explicitly named + // on the command line (like go install math). + // That's now handled by the isGoRelease clause above. + // The other effect of the shortcut was to isolate different entries in + // $GOPATH from each other. This had the unfortunate effect that + // if you had (say), GOPATH listing two entries, one for commands + // and one for libraries, and you did a 'git pull' in the library one + // and then tried 'go install commands/...', it would build the new libraries + // during the first build (because they wouldn't have been installed at all) + // but then subsequent builds would not rebuild the libraries, even if the + // mtimes indicate they are stale, because the different GOPATH entries + // were treated differently. This behavior was confusing when using + // non-trivial GOPATHs, which were particularly common with some + // code management conventions, like the original godep. + // Since the $GOROOT case (the original motivation) is handled separately, + // we no longer put a barrier between the different $GOPATH entries. + // + // One implication of this is that if there is a system directory for + // non-standard Go packages that is included in $GOPATH, the mtimes + // on those compiled packages must be no earlier than the mtimes + // on the source files. Since most distributions use the same mtime + // for all files in a tree, they will be unaffected. People using plain + // tar x to extract system-installed packages will need to adjust mtimes, + // but it's better to force them to get the mtimes right than to ignore + // the mtimes and thereby do the wrong thing in common use cases. + // + // So there is no GOPATH vs GOPATH shortcut here anymore. + // + // If something needs to come back here, we could try writing a dummy + // file with a random name to the $GOPATH/pkg directory (and removing it) + // to test for write access, and then skip GOPATH roots we don't have write + // access to. But hopefully we can just use the mtimes always. srcs := stringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) for _, src := range srcs { @@ -810,6 +1467,53 @@ func isStale(p *Package, topRoot map[string]bool) bool { return false } +// computeBuildID computes the build ID for p, leaving it in p.buildID. +// Build ID is a hash of the information we want to detect changes in. +// See the long comment in isStale for details. +func computeBuildID(p *Package) { + h := sha1.New() + + // Include the list of files compiled as part of the package. + // This lets us detect removed files. See issue 3895. + inputFiles := stringList( + p.GoFiles, + p.CgoFiles, + p.CFiles, + p.CXXFiles, + p.MFiles, + p.HFiles, + p.SFiles, + p.SysoFiles, + p.SwigFiles, + p.SwigCXXFiles, + ) + for _, file := range inputFiles { + fmt.Fprintf(h, "file %s\n", file) + } + + // Include the content of runtime/zversion.go in the hash + // for package runtime. This will give package runtime a + // different build ID in each Go release. + if p.Standard && p.ImportPath == "runtime" { + data, _ := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) + fmt.Fprintf(h, "zversion %q\n", string(data)) + } + + // Include the build IDs of any dependencies in the hash. + // This, combined with the runtime/zversion content, + // will cause packages to have different build IDs when + // compiled with different Go releases. + // This helps the go command know to recompile when + // people use the same GOPATH but switch between + // different Go releases. See issue 10702. + // This is also a better fix for issue 8290. + for _, p1 := range p.deps { + fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.buildID) + } + + p.buildID = fmt.Sprintf("%x", h.Sum(nil)) +} + var cwd, _ = os.Getwd() var cmdCache = map[string]*Package{} @@ -872,7 +1576,7 @@ func loadPackage(arg string, stk *importStack) *Package { } } - return loadImport(arg, cwd, stk, nil) + return loadImport(arg, cwd, nil, stk, nil, 0) } // packages returns the packages named by the @@ -942,6 +1646,23 @@ func packagesForBuild(args []string) []*Package { } } exitIfErrors() + + // Check for duplicate loads of the same package. + // That should be impossible, but if it does happen then + // we end up trying to build the same package twice, + // usually in parallel overwriting the same files, + // which doesn't work very well. + seen := map[string]bool{} + reported := map[string]bool{} + for _, pkg := range packageList(pkgs) { + if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { + reported[pkg.ImportPath] = true + errorf("internal error: duplicate loads of %s", pkg.ImportPath) + } + seen[pkg.ImportPath] = true + } + exitIfErrors() + return pkgs } @@ -967,3 +1688,170 @@ func hasSubdir(root, dir string) (rel string, ok bool) { } return filepath.ToSlash(dir[len(root):]), true } + +var ( + errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") + errBuildIDMalformed = fmt.Errorf("malformed object file") + errBuildIDUnknown = fmt.Errorf("lost build ID") +) + +var ( + bangArch = []byte("!<arch>") + pkgdef = []byte("__.PKGDEF") + goobject = []byte("go object ") + buildid = []byte("build id ") +) + +// readBuildID reads the build ID from an archive or binary. +// It only supports the gc toolchain. +// Other toolchain maintainers should adjust this function. +func readBuildID(p *Package) (id string, err error) { + if buildToolchain != (gcToolchain{}) { + return "", errBuildIDToolchain + } + + // For commands, read build ID directly from binary. + if p.Name == "main" { + return ReadBuildIDFromBinary(p.Target) + } + + // Otherwise, we expect to have an archive (.a) file, + // and we can read the build ID from the Go export data. + if !strings.HasSuffix(p.Target, ".a") { + return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDUnknown} + } + + // Read just enough of the target to fetch the build ID. + // The archive is expected to look like: + // + // !<arch> + // __.PKGDEF 0 0 0 644 7955 ` + // go object darwin amd64 devel X:none + // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" + // + // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). + // Reading the first 1024 bytes should be plenty. + f, err := os.Open(p.Target) + if err != nil { + return "", err + } + data := make([]byte, 1024) + n, err := io.ReadFull(f, data) + f.Close() + + if err != nil && n == 0 { + return "", err + } + + bad := func() (string, error) { + return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDMalformed} + } + + // Archive header. + for i := 0; ; i++ { // returns during i==3 + j := bytes.IndexByte(data, '\n') + if j < 0 { + return bad() + } + line := data[:j] + data = data[j+1:] + switch i { + case 0: + if !bytes.Equal(line, bangArch) { + return bad() + } + case 1: + if !bytes.HasPrefix(line, pkgdef) { + return bad() + } + case 2: + if !bytes.HasPrefix(line, goobject) { + return bad() + } + case 3: + if !bytes.HasPrefix(line, buildid) { + // Found the object header, just doesn't have a build id line. + // Treat as successful, with empty build id. + return "", nil + } + id, err := strconv.Unquote(string(line[len(buildid):])) + if err != nil { + return bad() + } + return id, nil + } + } +} + +var ( + goBuildPrefix = []byte("\xff Go build ID: \"") + goBuildEnd = []byte("\"\n \xff") + + elfPrefix = []byte("\x7fELF") +) + +// ReadBuildIDFromBinary reads the build ID from a binary. +// +// ELF binaries store the build ID in a proper PT_NOTE section. +// +// Other binary formats are not so flexible. For those, the linker +// stores the build ID as non-instruction bytes at the very beginning +// of the text segment, which should appear near the beginning +// of the file. This is clumsy but fairly portable. Custom locations +// can be added for other binary types as needed, like we did for ELF. +func ReadBuildIDFromBinary(filename string) (id string, err error) { + if filename == "" { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} + } + + // Read the first 16 kB of the binary file. + // That should be enough to find the build ID. + // In ELF files, the build ID is in the leading headers, + // which are typically less than 4 kB, not to mention 16 kB. + // On other systems, we're trying to read enough that + // we get the beginning of the text segment in the read. + // The offset where the text segment begins in a hello + // world compiled for each different object format today: + // + // Plan 9: 0x20 + // Windows: 0x600 + // Mach-O: 0x2000 + // + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + + data := make([]byte, 16*1024) + _, err = io.ReadFull(f, data) + if err == io.ErrUnexpectedEOF { + err = nil + } + if err != nil { + return "", err + } + + if bytes.HasPrefix(data, elfPrefix) { + return readELFGoBuildID(filename, f, data) + } + + i := bytes.Index(data, goBuildPrefix) + if i < 0 { + // Missing. Treat as successful but build ID empty. + return "", nil + } + + j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) + if j < 0 { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] + id, err = strconv.Unquote(string(quoted)) + if err != nil { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + return id, nil +} diff --git a/libgo/go/cmd/go/run.go b/libgo/go/cmd/go/run.go index ef8aa95a351..f6da373e252 100644 --- a/libgo/go/cmd/go/run.go +++ b/libgo/go/cmd/go/run.go @@ -37,7 +37,8 @@ Run compiles and runs the main package comprising the named Go source files. A Go source file is defined to be a file ending in a literal ".go" suffix. By default, 'go run' runs the compiled binary directly: 'a.out arguments...'. -If the -exec flag is given, 'go run' invokes the binary using xprog: 'xprog a.out arguments...'. +If the -exec flag is given, 'go run' invokes the binary using xprog: + 'xprog a.out arguments...'. If the -exec flag is not given, GOOS or GOARCH is different from the system default, and a program named go_$GOOS_$GOARCH_exec can be found on the current search path, 'go run' invokes the binary using that program, @@ -64,6 +65,7 @@ func printStderr(args ...interface{}) (int, error) { func runRun(cmd *Command, args []string) { raceInit() + buildModeInit() var b builder b.init() b.print = printStderr @@ -136,6 +138,7 @@ func runStdin(cmdline []string) { cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + cmd.Env = origEnv startSigHandlers() if err := cmd.Run(); err != nil { errorf("%v", err) diff --git a/libgo/go/cmd/go/test.go b/libgo/go/cmd/go/test.go index 5cf7aaf0716..aadfdf67cce 100644 --- a/libgo/go/cmd/go/test.go +++ b/libgo/go/cmd/go/test.go @@ -33,9 +33,11 @@ func init() { cmdTest.Run = runTest } +const testUsage = "test [-c] [-i] [build and test flags] [packages] [flags for test binary]" + var cmdTest = &Command{ CustomFlags: true, - UsageLine: "test [-c] [-i] [build and test flags] [packages] [flags for test binary]", + UsageLine: testUsage, Short: "test packages", Long: ` 'Go test' automates testing the packages named by the import paths. @@ -64,6 +66,21 @@ with source in the current directory, including tests, and runs the tests. The package is built in a temporary directory so it does not interfere with the non-test installation. +` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details. + +If the test binary needs any other flags, they should be presented after the +package names. The go tool treats as a flag the first argument that begins with +a minus sign that it does not recognize itself; that argument and all subsequent +arguments are passed as arguments to the test binary. + +For more about build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go vet. +`, +} + +const testFlag1 = ` In addition to the build flags, the flags handled by 'go test' itself are: -c @@ -83,21 +100,9 @@ In addition to the build flags, the flags handled by 'go test' itself are: Compile the test binary to the named file. The test still runs (unless -c or -i is specified). - The test binary also accepts flags that control execution of the test; these -flags are also accessible by 'go test'. See 'go help testflag' for details. - -If the test binary needs any other flags, they should be presented after the -package names. The go tool treats as a flag the first argument that begins with -a minus sign that it does not recognize itself; that argument and all subsequent -arguments are passed as arguments to the test binary. - -For more about build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go vet. -`, -} +flags are also accessible by 'go test'. +` var helpTestflag = &Command{ UsageLine: "testflag", @@ -107,13 +112,18 @@ The 'go test' command takes both flags that apply to 'go test' itself and flags that apply to the resulting test binary. Several of the flags control profiling and write an execution profile -suitable for "go tool pprof"; run "go tool pprof help" for more +suitable for "go tool pprof"; run "go tool pprof -h" for more information. The --alloc_space, --alloc_objects, and --show_bytes options of pprof control how the information is presented. The following flags are recognized by the 'go test' command and control the execution of any test: + ` + strings.TrimSpace(testFlag2) + ` +`, +} + +const testFlag2 = ` -bench regexp Run benchmarks matching the regular expression. By default, no benchmarks run. To run all benchmarks, @@ -135,12 +145,17 @@ control the execution of any test: -blockprofilerate n Control the detail provided in goroutine blocking profiles by calling runtime.SetBlockProfileRate with n. - See 'godoc runtime SetBlockProfileRate'. + See 'go doc runtime.SetBlockProfileRate'. The profiler aims to sample, on average, one blocking event every n nanoseconds the program spends blocked. By default, if -test.blockprofile is set without this flag, all blocking events are recorded, equivalent to -test.blockprofilerate=1. + -count n + Run each test and benchmark n times (default 1). + If -cpu is set, run n times for each GOMAXPROCS value. + Examples are always run once. + -cover Enable coverage analysis. @@ -180,7 +195,7 @@ control the execution of any test: -memprofilerate n Enable more precise (and expensive) memory profiles by setting - runtime.MemProfileRate. See 'godoc runtime MemProfileRate'. + runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. To profile all memory allocations, use -test.memprofilerate=1 and pass --alloc_space flag to the pprof tool. @@ -205,6 +220,11 @@ control the execution of any test: -timeout t If a test runs longer than t, panic. + The default is 10 minutes (10m). + + -trace trace.out + Write an execution trace to the specified file before exiting. + Writes test binary as -c would. -v Verbose output: log all tests as they are run. Also print all @@ -229,8 +249,7 @@ The test flags that generate profiles (other than for coverage) also leave the test binary in pkg.test for use when analyzing the profiles. Flags not recognized by 'go test' must be placed after any specified packages. -`, -} +` var helpTestfunc = &Command{ UsageLine: "testfunc", @@ -310,6 +329,7 @@ func runTest(cmd *Command, args []string) { findExecCmd() // initialize cached result raceInit() + buildModeInit() pkgs := packagesForBuild(pkgArgs) if len(pkgs) == 0 { fatalf("no packages to test") @@ -342,11 +362,11 @@ func runTest(cmd *Command, args []string) { // been given on the command line (implicit current directory) // or when benchmarking. // Also stream if we're showing output anyway with a - // single package under test. In that case, streaming the - // output produces the same result as not streaming, - // just more immediately. + // single package under test or if parallelism is set to 1. + // In these cases, streaming the output produces the same result + // as not streaming, just more immediately. testStreamOutput = len(pkgArgs) == 0 || testBench || - (len(pkgs) <= 1 && testShowPass) + (testShowPass && (len(pkgs) == 1 || buildP == 1)) var b builder b.init() @@ -364,10 +384,10 @@ func runTest(cmd *Command, args []string) { for _, path := range p.Imports { deps[path] = true } - for _, path := range p.TestImports { + for _, path := range p.vendored(p.TestImports) { deps[path] = true } - for _, path := range p.XTestImports { + for _, path := range p.vendored(p.XTestImports) { deps[path] = true } } @@ -376,7 +396,7 @@ func runTest(cmd *Command, args []string) { if deps["C"] { delete(deps, "C") deps["runtime/cgo"] = true - if buildContext.GOOS == runtime.GOOS && buildContext.GOARCH == runtime.GOARCH { + if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace { deps["cmd/cgo"] = true } } @@ -428,6 +448,10 @@ func runTest(cmd *Command, args []string) { // Mark all the coverage packages for rebuilding with coverage. for _, p := range testCoverPkgs { + // There is nothing to cover in package unsafe; it comes from the compiler. + if p.ImportPath == "unsafe" { + continue + } p.Stale = true // rebuild p.fake = true // do not warn about rebuild p.coverMode = testCoverMode @@ -562,15 +586,20 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, var imports, ximports []*Package var stk importStack stk.push(p.ImportPath + " (test)") - for _, path := range p.TestImports { - p1 := loadImport(path, p.Dir, &stk, p.build.TestImportPos[path]) + for i, path := range p.TestImports { + p1 := loadImport(path, p.Dir, p, &stk, p.build.TestImportPos[path], useVendor) if !reqStdPkgSrc && p1.Standard { continue } if p1.Error != nil { return nil, nil, nil, p1.Error } - if contains(p1.Deps, p.ImportPath) { + if len(p1.DepsErrors) > 0 { + err := p1.DepsErrors[0] + err.Pos = "" // show full import stack + return nil, nil, nil, err + } + if contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the // non-test copy of a package. @@ -581,24 +610,31 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, } return nil, nil, nil, err } + p.TestImports[i] = p1.ImportPath imports = append(imports, p1) } stk.pop() stk.push(p.ImportPath + "_test") pxtestNeedsPtest := false - for _, path := range p.XTestImports { - if path == p.ImportPath { - pxtestNeedsPtest = true - continue - } - p1 := loadImport(path, p.Dir, &stk, p.build.XTestImportPos[path]) + for i, path := range p.XTestImports { + p1 := loadImport(path, p.Dir, p, &stk, p.build.XTestImportPos[path], useVendor) if !reqStdPkgSrc && p1.Standard { continue } if p1.Error != nil { return nil, nil, nil, p1.Error } - ximports = append(ximports, p1) + if len(p1.DepsErrors) > 0 { + err := p1.DepsErrors[0] + err.Pos = "" // show full import stack + return nil, nil, nil, err + } + if p1.ImportPath == p.ImportPath { + pxtestNeedsPtest = true + } else { + ximports = append(ximports, p1) + } + p.XTestImports[i] = p1.ImportPath } stk.pop() @@ -723,7 +759,7 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, if dep == ptest.ImportPath { pmain.imports = append(pmain.imports, ptest) } else { - p1 := loadImport(dep, "", &stk, nil) + p1 := loadImport(dep, "", nil, &stk, nil, 0) if !reqStdPkgSrc && p1.Standard { continue } @@ -781,6 +817,12 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, recompileForTest(pmain, p, ptest, testDir) } + if buildContext.GOOS == "darwin" { + if buildContext.GOARCH == "arm" || buildContext.GOARCH == "arm64" { + t.NeedCgo = true + } + } + for _, cp := range pmain.imports { if len(cp.coverVars) > 0 { t.Cover = append(t.Cover, coverInfo{cp, cp.coverVars}) @@ -997,7 +1039,7 @@ func (b *builder) runTest(a *action) error { cmd := exec.Command(args[0], args[1:]...) cmd.Dir = a.p.Dir - cmd.Env = envForDir(cmd.Dir) + cmd.Env = envForDir(cmd.Dir, origEnv) var buf bytes.Buffer if testStreamOutput { cmd.Stdout = os.Stdout @@ -1216,6 +1258,7 @@ type testFuncs struct { NeedTest bool ImportXtest bool NeedXtest bool + NeedCgo bool Cover []coverInfo } @@ -1319,6 +1362,10 @@ import ( {{range $i, $p := .Cover}} _cover{{$i}} {{$p.Package.ImportPath | printf "%q"}} {{end}} + +{{if .NeedCgo}} + _ "runtime/cgo" +{{end}} ) var tests = []testing.InternalTest{ diff --git a/libgo/go/cmd/go/testdata/generate/test3.go b/libgo/go/cmd/go/testdata/generate/test3.go index 41ffb7ea87f..3d6a8a5c742 100644 --- a/libgo/go/cmd/go/testdata/generate/test3.go +++ b/libgo/go/cmd/go/testdata/generate/test3.go @@ -4,6 +4,6 @@ // Test go generate variable substitution. -//go:generate echo $GOARCH $GOFILE $GOPACKAGE xyz$GOPACKAGE/$GOFILE/123 +//go:generate echo $GOARCH $GOFILE:$GOLINE ${GOPACKAGE}abc xyz$GOPACKAGE/$GOFILE/123 package p diff --git a/libgo/go/cmd/go/testdata/generate/test4.go b/libgo/go/cmd/go/testdata/generate/test4.go new file mode 100644 index 00000000000..a7631c4a456 --- /dev/null +++ b/libgo/go/cmd/go/testdata/generate/test4.go @@ -0,0 +1,10 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test -run flag + +//go:generate echo oh yes my man +//go:generate echo no, no, a thousand times no + +package p diff --git a/libgo/go/cmd/go/testdata/rundir/sub/sub.go b/libgo/go/cmd/go/testdata/rundir/sub/sub.go new file mode 100644 index 00000000000..06ab7d0f9a3 --- /dev/null +++ b/libgo/go/cmd/go/testdata/rundir/sub/sub.go @@ -0,0 +1 @@ +package main diff --git a/libgo/go/cmd/go/testdata/rundir/x.go b/libgo/go/cmd/go/testdata/rundir/x.go new file mode 100644 index 00000000000..06ab7d0f9a3 --- /dev/null +++ b/libgo/go/cmd/go/testdata/rundir/x.go @@ -0,0 +1 @@ +package main diff --git a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go new file mode 100644 index 00000000000..7a471f0cc05 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1.go @@ -0,0 +1 @@ +package q1 diff --git a/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go new file mode 100644 index 00000000000..ca81bd2bf80 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testcycle/q1/q1_test.go @@ -0,0 +1,6 @@ +package q1 + +import "testing" +import _ "testcycle/q1" + +func Test(t *testing.T) {} diff --git a/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go b/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go new file mode 100644 index 00000000000..a457035a430 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testdep/p1/p1.go @@ -0,0 +1 @@ +package p1 diff --git a/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go b/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go new file mode 100644 index 00000000000..8be75334425 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testdep/p1/p1_test.go @@ -0,0 +1,3 @@ +package p1 + +import _ "testdep/p2" diff --git a/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go b/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go new file mode 100644 index 00000000000..15ba2eacea5 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testdep/p2/p2.go @@ -0,0 +1,3 @@ +package p2 + +import _ "testdep/p3" diff --git a/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go b/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go new file mode 100644 index 00000000000..0219e7fae50 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testdep/p3/p3.go @@ -0,0 +1,3 @@ +// +build ignore + +package ignored diff --git a/libgo/go/cmd/go/testdata/src/vend/bad.go b/libgo/go/cmd/go/testdata/src/vend/bad.go new file mode 100644 index 00000000000..57cc595220c --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/bad.go @@ -0,0 +1,3 @@ +package vend + +import _ "r" diff --git a/libgo/go/cmd/go/testdata/src/vend/good.go b/libgo/go/cmd/go/testdata/src/vend/good.go new file mode 100644 index 00000000000..952ada3108d --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/good.go @@ -0,0 +1,3 @@ +package vend + +import _ "p" diff --git a/libgo/go/cmd/go/testdata/src/vend/hello/hello.go b/libgo/go/cmd/go/testdata/src/vend/hello/hello.go new file mode 100644 index 00000000000..41dc03e0ce4 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/hello/hello.go @@ -0,0 +1,10 @@ +package main + +import ( + "fmt" + "strings" // really ../vendor/strings +) + +func main() { + fmt.Printf("%s\n", strings.Msg) +} diff --git a/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go b/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go new file mode 100644 index 00000000000..5e72ada9387 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go @@ -0,0 +1,12 @@ +package main + +import ( + "strings" // really ../vendor/strings + "testing" +) + +func TestMsgInternal(t *testing.T) { + if strings.Msg != "hello, world" { + t.Fatal("unexpected msg: %v", strings.Msg) + } +} diff --git a/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go b/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go new file mode 100644 index 00000000000..96e6049dad0 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go @@ -0,0 +1,12 @@ +package main_test + +import ( + "strings" // really ../vendor/strings + "testing" +) + +func TestMsgExternal(t *testing.T) { + if strings.Msg != "hello, world" { + t.Fatal("unexpected msg: %v", strings.Msg) + } +} diff --git a/libgo/go/cmd/go/testdata/src/vend/subdir/bad.go b/libgo/go/cmd/go/testdata/src/vend/subdir/bad.go new file mode 100644 index 00000000000..d0ddaacfea5 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/subdir/bad.go @@ -0,0 +1,3 @@ +package subdir + +import _ "r" diff --git a/libgo/go/cmd/go/testdata/src/vend/subdir/good.go b/libgo/go/cmd/go/testdata/src/vend/subdir/good.go new file mode 100644 index 00000000000..edd04543a2b --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/subdir/good.go @@ -0,0 +1,3 @@ +package subdir + +import _ "p" diff --git a/libgo/go/cmd/go/testdata/src/vend/vendor/p/p.go b/libgo/go/cmd/go/testdata/src/vend/vendor/p/p.go new file mode 100644 index 00000000000..c89cd18d0fe --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/vendor/p/p.go @@ -0,0 +1 @@ +package p diff --git a/libgo/go/cmd/go/testdata/src/vend/vendor/q/q.go b/libgo/go/cmd/go/testdata/src/vend/vendor/q/q.go new file mode 100644 index 00000000000..946e6d99109 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/vendor/q/q.go @@ -0,0 +1 @@ +package q diff --git a/libgo/go/cmd/go/testdata/src/vend/vendor/strings/msg.go b/libgo/go/cmd/go/testdata/src/vend/vendor/strings/msg.go new file mode 100644 index 00000000000..438126ba2be --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/vendor/strings/msg.go @@ -0,0 +1,3 @@ +package strings + +var Msg = "hello, world" diff --git a/libgo/go/cmd/go/testdata/src/vend/x/invalid/invalid.go b/libgo/go/cmd/go/testdata/src/vend/x/invalid/invalid.go new file mode 100644 index 00000000000..e250d5bb31b --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/x/invalid/invalid.go @@ -0,0 +1,3 @@ +package invalid + +import "vend/x/invalid/vendor/foo" diff --git a/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p.go b/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p.go new file mode 100644 index 00000000000..c89cd18d0fe --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p.go @@ -0,0 +1 @@ +package p diff --git a/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p/p.go b/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p/p.go new file mode 100644 index 00000000000..e12e12c2f4c --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/x/vendor/p/p/p.go @@ -0,0 +1,3 @@ +package p + +import _ "notfound" diff --git a/libgo/go/cmd/go/testdata/src/vend/x/vendor/r/r.go b/libgo/go/cmd/go/testdata/src/vend/x/vendor/r/r.go new file mode 100644 index 00000000000..838c177a570 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/x/vendor/r/r.go @@ -0,0 +1 @@ +package r diff --git a/libgo/go/cmd/go/testdata/src/vend/x/x.go b/libgo/go/cmd/go/testdata/src/vend/x/x.go new file mode 100644 index 00000000000..ae526ebdda2 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/x/x.go @@ -0,0 +1,5 @@ +package x + +import _ "p" +import _ "q" +import _ "r" diff --git a/libgo/go/cmd/go/testdata/src/vetpkg/c.go b/libgo/go/cmd/go/testdata/src/vetpkg/c.go new file mode 100644 index 00000000000..ef5648f0590 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vetpkg/c.go @@ -0,0 +1,9 @@ +// +build tagtest + +package p + +import "fmt" + +func g() { + fmt.Printf("%d", 3, 4) +} diff --git a/libgo/go/cmd/go/testdata/testinternal3/t.go b/libgo/go/cmd/go/testdata/testinternal3/t.go new file mode 100644 index 00000000000..8576a4b4d76 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testinternal3/t.go @@ -0,0 +1,3 @@ +package t + +import _ "internal/does-not-exist" diff --git a/libgo/go/cmd/go/testdata/testinternal4/src/p/p.go b/libgo/go/cmd/go/testdata/testinternal4/src/p/p.go new file mode 100644 index 00000000000..6bdee27be2f --- /dev/null +++ b/libgo/go/cmd/go/testdata/testinternal4/src/p/p.go @@ -0,0 +1,6 @@ +package p + +import ( + _ "q/internal/x" + _ "q/j" +) diff --git a/libgo/go/cmd/go/testdata/testinternal4/src/q/internal/x/x.go b/libgo/go/cmd/go/testdata/testinternal4/src/q/internal/x/x.go new file mode 100644 index 00000000000..823aafd0712 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testinternal4/src/q/internal/x/x.go @@ -0,0 +1 @@ +package x diff --git a/libgo/go/cmd/go/testdata/testinternal4/src/q/j/j.go b/libgo/go/cmd/go/testdata/testinternal4/src/q/j/j.go new file mode 100644 index 00000000000..9f075438940 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testinternal4/src/q/j/j.go @@ -0,0 +1,3 @@ +package j + +import _ "q/internal/x" diff --git a/libgo/go/cmd/go/testdata/testvendor/src/p/p.go b/libgo/go/cmd/go/testdata/testvendor/src/p/p.go new file mode 100644 index 00000000000..e740715186e --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor/src/p/p.go @@ -0,0 +1,6 @@ +package p + +import ( + _ "q/y" + _ "q/z" +) diff --git a/libgo/go/cmd/go/testdata/testvendor/src/q/vendor/x/x.go b/libgo/go/cmd/go/testdata/testvendor/src/q/vendor/x/x.go new file mode 100644 index 00000000000..823aafd0712 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor/src/q/vendor/x/x.go @@ -0,0 +1 @@ +package x diff --git a/libgo/go/cmd/go/testdata/testvendor/src/q/y/y.go b/libgo/go/cmd/go/testdata/testvendor/src/q/y/y.go new file mode 100644 index 00000000000..4f842237675 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor/src/q/y/y.go @@ -0,0 +1,3 @@ +package y + +import _ "x" diff --git a/libgo/go/cmd/go/testdata/testvendor/src/q/z/z.go b/libgo/go/cmd/go/testdata/testvendor/src/q/z/z.go new file mode 100644 index 00000000000..a8d4924936a --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor/src/q/z/z.go @@ -0,0 +1,3 @@ +package z + +import _ "q/vendor/x" diff --git a/libgo/go/cmd/go/testdata/testvendor2/src/p/p.go b/libgo/go/cmd/go/testdata/testvendor2/src/p/p.go new file mode 100644 index 00000000000..220b2b2a071 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor2/src/p/p.go @@ -0,0 +1,3 @@ +package p + +import "x" diff --git a/libgo/go/cmd/go/testdata/testvendor2/vendor/x/x.go b/libgo/go/cmd/go/testdata/testvendor2/vendor/x/x.go new file mode 100644 index 00000000000..823aafd0712 --- /dev/null +++ b/libgo/go/cmd/go/testdata/testvendor2/vendor/x/x.go @@ -0,0 +1 @@ +package x diff --git a/libgo/go/cmd/go/testflag.go b/libgo/go/cmd/go/testflag.go index 6da74b99677..1f3e3d316af 100644 --- a/libgo/go/cmd/go/testflag.go +++ b/libgo/go/cmd/go/testflag.go @@ -5,6 +5,7 @@ package main import ( + "flag" "fmt" "os" "strconv" @@ -16,46 +17,11 @@ import ( // our command line are for us, and some are for 6.out, and // some are for both. -var usageMessage = `Usage of go test: - -c=false: compile but do not run the test binary - -file=file_test.go: specify file to use for tests; - use multiple times for multiple files - -p=n: build and test up to n packages in parallel - -x=false: print command lines as they are executed - - // These flags can be passed with or without a "test." prefix: -v or -test.v. - -bench="": passes -test.bench to test - -benchmem=false: print memory allocation statistics for benchmarks - -benchtime=1s: passes -test.benchtime to test - -cover=false: enable coverage analysis - -covermode="set": specifies mode for coverage analysis - -coverpkg="": comma-separated list of packages for coverage analysis - -coverprofile="": passes -test.coverprofile to test if -cover - -cpu="": passes -test.cpu to test - -cpuprofile="": passes -test.cpuprofile to test - -memprofile="": passes -test.memprofile to test - -memprofilerate=0: passes -test.memprofilerate to test - -blockprofile="": pases -test.blockprofile to test - -blockprofilerate=0: passes -test.blockprofilerate to test - -outputdir=$PWD: passes -test.outputdir to test - -parallel=0: passes -test.parallel to test - -run="": passes -test.run to test - -short=false: passes -test.short to test - -timeout=0: passes -test.timeout to test - -v=false: passes -test.v to test -` - -// usage prints a usage message and exits. -func testUsage() { - fmt.Fprint(os.Stderr, usageMessage) - setExitStatus(2) - exit() -} - // testFlagSpec defines a flag we know about. type testFlagSpec struct { name string boolVar *bool + flagValue flag.Value passToTest bool // pass to Test multiOK bool // OK to have multiple instances present bool // flag has been seen @@ -65,32 +31,18 @@ type testFlagSpec struct { var testFlagDefn = []*testFlagSpec{ // local. {name: "c", boolVar: &testC}, + {name: "i", boolVar: &buildI}, + {name: "o"}, {name: "cover", boolVar: &testCover}, + {name: "covermode"}, {name: "coverpkg"}, - {name: "o"}, - - // build flags. - {name: "a", boolVar: &buildA}, - {name: "n", boolVar: &buildN}, - {name: "p"}, - {name: "x", boolVar: &buildX}, - {name: "i", boolVar: &buildI}, - {name: "work", boolVar: &buildWork}, - {name: "ccflags"}, - {name: "gcflags"}, {name: "exec"}, - {name: "ldflags"}, - {name: "gccgoflags"}, - {name: "tags"}, - {name: "compiler"}, - {name: "race", boolVar: &buildRace}, - {name: "installsuffix"}, // passed to 6.out, adding a "test." prefix to the name if necessary: -v becomes -test.v. {name: "bench", passToTest: true}, {name: "benchmem", boolVar: new(bool), passToTest: true}, {name: "benchtime", passToTest: true}, - {name: "covermode"}, + {name: "count", passToTest: true}, {name: "coverprofile", passToTest: true}, {name: "cpu", passToTest: true}, {name: "cpuprofile", passToTest: true}, @@ -103,9 +55,26 @@ var testFlagDefn = []*testFlagSpec{ {name: "run", passToTest: true}, {name: "short", boolVar: new(bool), passToTest: true}, {name: "timeout", passToTest: true}, + {name: "trace", passToTest: true}, {name: "v", boolVar: &testV, passToTest: true}, } +// add build flags to testFlagDefn +func init() { + var cmd Command + addBuildFlags(&cmd) + cmd.Flag.VisitAll(func(f *flag.Flag) { + if f.Name == "v" { + // test overrides the build -v flag + return + } + testFlagDefn = append(testFlagDefn, &testFlagSpec{ + name: f.Name, + flagValue: f.Value, + }) + }) +} + // testFlags processes the command line, grabbing -x and -c, rewriting known flags // to have "test" before them, and reading the command line for the 6.out. // Unfortunately for us, we need to do our own flag processing because go test @@ -148,73 +117,55 @@ func testFlags(args []string) (packageNames, passToTest []string) { passToTest = append(passToTest, args[i]) continue } - var err error - switch f.name { - // bool flags. - case "a", "c", "i", "n", "x", "v", "race", "cover", "work": - setBoolFlag(f.boolVar, value) - case "o": - testO = value - testNeedBinary = true - case "p": - setIntFlag(&buildP, value) - case "exec": - execCmd, err = splitQuotedFields(value) - if err != nil { + if f.flagValue != nil { + if err := f.flagValue.Set(value); err != nil { fatalf("invalid flag argument for -%s: %v", f.name, err) } - case "ccflags": - buildCcflags, err = splitQuotedFields(value) - if err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - case "gcflags": - buildGcflags, err = splitQuotedFields(value) - if err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - case "ldflags": - buildLdflags, err = splitQuotedFields(value) - if err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - case "gccgoflags": - buildGccgoflags, err = splitQuotedFields(value) - if err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - case "tags": - buildContext.BuildTags = strings.Fields(value) - case "compiler": - buildCompiler{}.Set(value) - case "bench": - // record that we saw the flag; don't care about the value - testBench = true - case "timeout": - testTimeout = value - case "blockprofile", "cpuprofile", "memprofile": - testProfile = true - testNeedBinary = true - case "coverpkg": - testCover = true - if value == "" { - testCoverPaths = nil - } else { - testCoverPaths = strings.Split(value, ",") - } - case "coverprofile": - testCover = true - testProfile = true - case "covermode": - switch value { - case "set", "count", "atomic": - testCoverMode = value - default: - fatalf("invalid flag argument for -cover: %q", value) + } else { + // Test-only flags. + // Arguably should be handled by f.flagValue, but aren't. + var err error + switch f.name { + // bool flags. + case "c", "i", "v", "cover": + setBoolFlag(f.boolVar, value) + case "o": + testO = value + testNeedBinary = true + case "exec": + execCmd, err = splitQuotedFields(value) + if err != nil { + fatalf("invalid flag argument for -%s: %v", f.name, err) + } + case "bench": + // record that we saw the flag; don't care about the value + testBench = true + case "timeout": + testTimeout = value + case "blockprofile", "cpuprofile", "memprofile", "trace": + testProfile = true + testNeedBinary = true + case "coverpkg": + testCover = true + if value == "" { + testCoverPaths = nil + } else { + testCoverPaths = strings.Split(value, ",") + } + case "coverprofile": + testCover = true + testProfile = true + case "covermode": + switch value { + case "set", "count", "atomic": + testCoverMode = value + default: + fatalf("invalid flag argument for -covermode: %q", value) + } + testCover = true + case "outputdir": + outputDir = value } - testCover = true - case "outputdir": - outputDir = value } if extraWord { i++ @@ -267,7 +218,7 @@ func testFlag(args []string, i int) (f *testFlagSpec, value string, extra bool) for _, f = range testFlagDefn { if name == f.name { // Booleans are special because they have modes -x, -x=true, -x=false. - if f.boolVar != nil { + if f.boolVar != nil || isBoolFlag(f.flagValue) { if equals < 0 { // otherwise, it's been set and will be verified in setBoolFlag value = "true" } else { @@ -294,6 +245,17 @@ func testFlag(args []string, i int) (f *testFlagSpec, value string, extra bool) return } +// isBoolFlag reports whether v is a bool flag. +func isBoolFlag(v flag.Value) bool { + vv, ok := v.(interface { + IsBoolFlag() bool + }) + if ok { + return vv.IsBoolFlag() + } + return false +} + // setBoolFlag sets the addressed boolean to the value. func setBoolFlag(flag *bool, value string) { x, err := strconv.ParseBool(value) diff --git a/libgo/go/cmd/go/tool.go b/libgo/go/cmd/go/tool.go index 3f11c3e3d44..937ca1f306c 100644 --- a/libgo/go/cmd/go/tool.go +++ b/libgo/go/cmd/go/tool.go @@ -50,6 +50,9 @@ func tool(toolName string) string { if toolIsWindows { toolPath += toolWindowsExtension } + if len(buildToolExec) > 0 { + return toolPath + } // Give a nice message if there is no tool with that name. if _, err := os.Stat(toolPath); err != nil { if isInGoToolsRepo(toolName) { @@ -64,10 +67,6 @@ func tool(toolName string) string { } func isInGoToolsRepo(toolName string) bool { - switch toolName { - case "cover", "vet": - return true - } return false } @@ -92,7 +91,11 @@ func runTool(cmd *Command, args []string) { return } if toolN { - fmt.Printf("%s %s\n", toolPath, strings.Join(args[1:], " ")) + cmd := toolPath + if len(args) > 1 { + cmd += " " + strings.Join(args[1:], " ") + } + fmt.Printf("%s\n", cmd) return } toolCmd := &exec.Cmd{ @@ -101,6 +104,8 @@ func runTool(cmd *Command, args []string) { Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, + // Set $GOROOT, mainly for go tool dist. + Env: mergeEnvLists([]string{"GOROOT=" + goroot}, os.Environ()), } err := toolCmd.Run() if err != nil { diff --git a/libgo/go/cmd/go/vcs.go b/libgo/go/cmd/go/vcs.go index 1cac6133889..28a7540dfe4 100644 --- a/libgo/go/cmd/go/vcs.go +++ b/libgo/go/cmd/go/vcs.go @@ -9,12 +9,15 @@ import ( "encoding/json" "errors" "fmt" + "internal/singleflight" "log" + "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" + "sync" ) // A vcsCmd describes how to use a version control system @@ -23,13 +26,13 @@ type vcsCmd struct { name string cmd string // name of binary to invoke command - createCmd string // command to download a fresh copy of a repository - downloadCmd string // command to download updates into an existing repository + createCmd []string // commands to download a fresh copy of a repository + downloadCmd []string // commands to download updates into an existing repository tagCmd []tagCmd // commands to list tags tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd string // command to sync to specific tag - tagSyncDefault string // command to sync to default tag + tagSyncCmd []string // commands to sync to specific tag + tagSyncDefault []string // commands to sync to default tag scheme []string pingCmd string @@ -38,6 +41,23 @@ type vcsCmd struct { resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) } +var isSecureScheme = map[string]bool{ + "https": true, + "git+ssh": true, + "bzr+ssh": true, + "svn+ssh": true, + "ssh": true, +} + +func (v *vcsCmd) isSecure(repo string) bool { + u, err := url.Parse(repo) + if err != nil { + // If repo is not a URL, it's not secure. + return false + } + return isSecureScheme[u.Scheme] +} + // A tagCmd describes a command to list available tags // that can be passed to tagSyncCmd. type tagCmd struct { @@ -69,8 +89,8 @@ var vcsHg = &vcsCmd{ name: "Mercurial", cmd: "hg", - createCmd: "clone -U {repo} {dir}", - downloadCmd: "pull", + createCmd: []string{"clone -U {repo} {dir}"}, + downloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' // for selecting a version. This lets people have @@ -81,8 +101,8 @@ var vcsHg = &vcsCmd{ {"tags", `^(\S+)`}, {"branches", `^(\S+)`}, }, - tagSyncCmd: "update -r {tag}", - tagSyncDefault: "update default", + tagSyncCmd: []string{"update -r {tag}"}, + tagSyncDefault: []string{"update default"}, scheme: []string{"https", "http", "ssh"}, pingCmd: "identify {scheme}://{repo}", @@ -102,8 +122,8 @@ var vcsGit = &vcsCmd{ name: "Git", cmd: "git", - createCmd: "clone {repo} {dir}", - downloadCmd: "pull --ff-only", + createCmd: []string{"clone {repo} {dir}", "--git-dir={dir}/.git submodule update --init --recursive"}, + downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, tagCmd: []tagCmd{ // tags/xxx matches a git tag named xxx @@ -113,41 +133,64 @@ var vcsGit = &vcsCmd{ tagLookupCmd: []tagCmd{ {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, }, - tagSyncCmd: "checkout {tag}", - tagSyncDefault: "checkout master", - - scheme: []string{"git", "https", "http", "git+ssh"}, + tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + // both createCmd and downloadCmd update the working dir. + // No need to do more here. We used to 'checkout master' + // but that doesn't work if the default branch is not named master. + // See golang.org/issue/9032. + tagSyncDefault: []string{"checkout master", "submodule update --init --recursive"}, + + scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, pingCmd: "ls-remote {scheme}://{repo}", remoteRepo: gitRemoteRepo, } +// scpSyntaxRe matches the SCP-like addresses used by Git to access +// repositories by SSH. +var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) + func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { - outb, err := vcsGit.runOutput(rootDir, "remote -v") + cmd := "config remote.origin.url" + errParse := errors.New("unable to parse output of git " + cmd) + errRemoteOriginNotFound := errors.New("remote origin not found") + outb, err := vcsGit.run1(rootDir, cmd, nil, false) if err != nil { + // if it doesn't output any message, it means the config argument is correct, + // but the config value itself doesn't exist + if outb != nil && len(outb) == 0 { + return "", errRemoteOriginNotFound + } return "", err } - out := string(outb) - - // Expect: - // origin https://github.com/rsc/pdf (fetch) - // origin https://github.com/rsc/pdf (push) - // use first line only. - - if !strings.HasPrefix(out, "origin\t") { - return "", fmt.Errorf("unable to parse output of git remote -v") - } - out = strings.TrimPrefix(out, "origin\t") - i := strings.Index(out, "\n") - if i < 0 { - return "", fmt.Errorf("unable to parse output of git remote -v") + out := strings.TrimSpace(string(outb)) + + var repoURL *url.URL + if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { + // Match SCP-like syntax and convert it to a URL. + // Eg, "git@github.com:user/repo" becomes + // "ssh://git@github.com/user/repo". + repoURL = &url.URL{ + Scheme: "ssh", + User: url.User(m[1]), + Host: m[2], + RawPath: m[3], + } + } else { + repoURL, err = url.Parse(out) + if err != nil { + return "", err + } } - out = out[:i] - i = strings.LastIndex(out, " ") - if i < 0 { - return "", fmt.Errorf("unable to parse output of git remote -v") + + // Iterate over insecure schemes too, because this function simply + // reports the state of the repo. If we can't see insecure schemes then + // we can't report the actual repo URL. + for _, s := range vcsGit.scheme { + if repoURL.Scheme == s { + return repoURL.String(), nil + } } - out = out[:i] - return strings.TrimSpace(string(out)), nil + return "", errParse } // vcsBzr describes how to use Bazaar. @@ -155,15 +198,15 @@ var vcsBzr = &vcsCmd{ name: "Bazaar", cmd: "bzr", - createCmd: "branch {repo} {dir}", + createCmd: []string{"branch {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: "pull --overwrite", + downloadCmd: []string{"pull --overwrite"}, tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: "update -r {tag}", - tagSyncDefault: "update -r revno:-1", + tagSyncCmd: []string{"update -r {tag}"}, + tagSyncDefault: []string{"update -r revno:-1"}, scheme: []string{"https", "http", "bzr", "bzr+ssh"}, pingCmd: "info {scheme}://{repo}", @@ -217,8 +260,8 @@ var vcsSvn = &vcsCmd{ name: "Subversion", cmd: "svn", - createCmd: "checkout {repo} {dir}", - downloadCmd: "update", + createCmd: []string{"checkout {repo} {dir}"}, + downloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. @@ -294,14 +337,14 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) _, err := exec.LookPath(v.cmd) if err != nil { fmt.Fprintf(os.Stderr, - "go: missing %s command. See http://golang.org/s/gogetcmd\n", + "go: missing %s command. See https://golang.org/s/gogetcmd\n", v.name) return nil, err } cmd := exec.Command(v.cmd, args...) cmd.Dir = dir - cmd.Env = envForDir(cmd.Dir) + cmd.Env = envForDir(cmd.Dir, os.Environ()) if buildX { fmt.Printf("cd %s\n", dir) fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) @@ -316,7 +359,7 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) os.Stderr.Write(out) } - return nil, err + return out, err } return out, nil } @@ -329,7 +372,15 @@ func (v *vcsCmd) ping(scheme, repo string) error { // create creates a new copy of repo in dir. // The parent of dir must exist; dir must not. func (v *vcsCmd) create(dir, repo string) error { - return v.run(".", v.createCmd, "dir", dir, "repo", repo) + for _, cmd := range v.createCmd { + if !go15VendorExperiment && strings.Contains(cmd, "submodule") { + continue + } + if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { + return err + } + } + return nil } // download downloads any new changes for the repo in dir. @@ -337,7 +388,15 @@ func (v *vcsCmd) download(dir string) error { if err := v.fixDetachedHead(dir); err != nil { return err } - return v.run(dir, v.downloadCmd) + for _, cmd := range v.downloadCmd { + if !go15VendorExperiment && strings.Contains(cmd, "submodule") { + continue + } + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil } // fixDetachedHead switches a Git repository in dir from a detached head to the master branch. @@ -383,7 +442,7 @@ func (v *vcsCmd) tags(dir string) ([]string, error) { // tagSync syncs the repo in dir to the named tag, // which either is a tag returned by tags or is v.tagDefault. func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == "" { + if v.tagSyncCmd == nil { return nil } if tag != "" { @@ -400,10 +459,28 @@ func (v *vcsCmd) tagSync(dir, tag string) error { } } } - if tag == "" && v.tagSyncDefault != "" { - return v.run(dir, v.tagSyncDefault) + + if tag == "" && v.tagSyncDefault != nil { + for _, cmd := range v.tagSyncDefault { + if !go15VendorExperiment && strings.Contains(cmd, "submodule") { + continue + } + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil + } + + for _, cmd := range v.tagSyncCmd { + if !go15VendorExperiment && strings.Contains(cmd, "submodule") { + continue + } + if err := v.run(dir, cmd, "tag", tag); err != nil { + return err + } } - return v.run(dir, v.tagSyncCmd, "tag", tag) + return nil } // A vcsPath describes how to convert an import path into a @@ -467,10 +544,20 @@ type repoRoot struct { var httpPrefixRE = regexp.MustCompile(`^https?:`) +// securityMode specifies whether a function should make network +// calls using insecure transports (eg, plain text HTTP). +// The zero value is "secure". +type securityMode int + +const ( + secure securityMode = iota + insecure +) + // repoRootForImportPath analyzes importPath to determine the // version control system, and code repository to use. -func repoRootForImportPath(importPath string) (*repoRoot, error) { - rr, err := repoRootForImportPathStatic(importPath, "") +func repoRootForImportPath(importPath string, security securityMode) (*repoRoot, error) { + rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) if err == errUnknownSite { // If there are wildcards, look up the thing before the wildcard, // hoping it applies to the wildcarded parts too. @@ -479,7 +566,7 @@ func repoRootForImportPath(importPath string) (*repoRoot, error) { if i := strings.Index(lookup, "/.../"); i >= 0 { lookup = lookup[:i] } - rr, err = repoRootForImportDynamic(lookup) + rr, err = repoRootForImportDynamic(lookup, security) // repoRootForImportDynamic returns error detail // that is irrelevant if the user didn't intend to use a @@ -492,6 +579,13 @@ func repoRootForImportPath(importPath string) (*repoRoot, error) { err = fmt.Errorf("unrecognized import path %q", importPath) } } + if err != nil { + rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) + if err1 == nil { + rr = rr1 + err = nil + } + } if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { // Do not allow wildcards in the repo root. @@ -503,13 +597,10 @@ func repoRootForImportPath(importPath string) (*repoRoot, error) { var errUnknownSite = errors.New("dynamic lookup required to find mapping") -// repoRootForImportPathStatic attempts to map importPath to a -// repoRoot using the commonly-used VCS hosting sites in vcsPaths -// (github.com/user/dir), or from a fully-qualified importPath already -// containing its VCS type (foo.com/repo.git/dir) -// +// repoRootFromVCSPaths attempts to map importPath to a repoRoot +// using the mappings defined in vcsPaths. // If scheme is non-empty, that scheme is forced. -func repoRootForImportPathStatic(importPath, scheme string) (*repoRoot, error) { +func repoRootFromVCSPaths(importPath, scheme string, security securityMode, vcsPaths []*vcsPath) (*repoRoot, error) { // A common error is to use https://packagepath because that's what // hg and git require. Diagnose this helpfully. if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { @@ -559,6 +650,9 @@ func repoRootForImportPathStatic(importPath, scheme string) (*repoRoot, error) { match["repo"] = scheme + "://" + match["repo"] } else { for _, scheme := range vcs.scheme { + if security == secure && !isSecureScheme[scheme] { + continue + } if vcs.ping(scheme, match["repo"]) == nil { match["repo"] = scheme + "://" + match["repo"] break @@ -579,26 +673,31 @@ func repoRootForImportPathStatic(importPath, scheme string) (*repoRoot, error) { // repoRootForImportDynamic finds a *repoRoot for a custom domain that's not // statically known by repoRootForImportPathStatic. // -// This handles "vanity import paths" like "name.tld/pkg/foo". -func repoRootForImportDynamic(importPath string) (*repoRoot, error) { +// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". +func repoRootForImportDynamic(importPath string, security securityMode) (*repoRoot, error) { slash := strings.Index(importPath, "/") if slash < 0 { - return nil, errors.New("import path does not contain a slash") + slash = len(importPath) } host := importPath[:slash] if !strings.Contains(host, ".") { return nil, errors.New("import path does not begin with hostname") } - urlStr, body, err := httpsOrHTTP(importPath) + urlStr, body, err := httpsOrHTTP(importPath, security) if err != nil { - return nil, fmt.Errorf("http/https fetch: %v", err) + msg := "https fetch: %v" + if security == insecure { + msg = "http/" + msg + } + return nil, fmt.Errorf(msg, err) } defer body.Close() imports, err := parseMetaGoImports(body) if err != nil { return nil, fmt.Errorf("parsing %s: %v", importPath, err) } - metaImport, err := matchGoImport(imports, importPath) + // Find the matched meta import. + mmi, err := matchGoImport(imports, importPath) if err != nil { if err != errNoMatch { return nil, fmt.Errorf("parse %s: %v", urlStr, err) @@ -606,7 +705,7 @@ func repoRootForImportDynamic(importPath string) (*repoRoot, error) { return nil, fmt.Errorf("parse %s: no go-import meta tags", urlStr) } if buildV { - log.Printf("get %q: found meta tag %#v at %s", importPath, metaImport, urlStr) + log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) } // If the import was "uni.edu/bob/project", which said the // prefix was "uni.edu" and the RepoRoot was "evilroot.com", @@ -614,42 +713,89 @@ func repoRootForImportDynamic(importPath string) (*repoRoot, error) { // "uni.edu" yet (possibly overwriting/preempting another // non-evil student). Instead, first verify the root and see // if it matches Bob's claim. - if metaImport.Prefix != importPath { + if mmi.Prefix != importPath { if buildV { log.Printf("get %q: verifying non-authoritative meta tag", importPath) } urlStr0 := urlStr - urlStr, body, err = httpsOrHTTP(metaImport.Prefix) + var imports []metaImport + urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) if err != nil { - return nil, fmt.Errorf("fetch %s: %v", urlStr, err) - } - imports, err := parseMetaGoImports(body) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", importPath, err) - } - if len(imports) == 0 { - return nil, fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + return nil, err } metaImport2, err := matchGoImport(imports, importPath) - if err != nil || metaImport != metaImport2 { - return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, metaImport.Prefix) + if err != nil || mmi != metaImport2 { + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) } } - if !strings.Contains(metaImport.RepoRoot, "://") { - return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, metaImport.RepoRoot) + if !strings.Contains(mmi.RepoRoot, "://") { + return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) } rr := &repoRoot{ - vcs: vcsByCmd(metaImport.VCS), - repo: metaImport.RepoRoot, - root: metaImport.Prefix, + vcs: vcsByCmd(mmi.VCS), + repo: mmi.RepoRoot, + root: mmi.Prefix, } if rr.vcs == nil { - return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, metaImport.VCS) + return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) } return rr, nil } +var fetchGroup singleflight.Group +var ( + fetchCacheMu sync.Mutex + fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix +) + +// metaImportsForPrefix takes a package's root import path as declared in a <meta> tag +// and returns its HTML discovery URL and the parsed metaImport lines +// found on the page. +// +// The importPath is of the form "golang.org/x/tools". +// It is an error if no imports are found. +// urlStr will still be valid if err != nil. +// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" +func metaImportsForPrefix(importPrefix string, security securityMode) (urlStr string, imports []metaImport, err error) { + setCache := func(res fetchResult) (fetchResult, error) { + fetchCacheMu.Lock() + defer fetchCacheMu.Unlock() + fetchCache[importPrefix] = res + return res, nil + } + + resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { + fetchCacheMu.Lock() + if res, ok := fetchCache[importPrefix]; ok { + fetchCacheMu.Unlock() + return res, nil + } + fetchCacheMu.Unlock() + + urlStr, body, err := httpsOrHTTP(importPrefix, security) + if err != nil { + return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) + } + imports, err := parseMetaGoImports(body) + if err != nil { + return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) + } + if len(imports) == 0 { + err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + } + return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) + }) + res := resi.(fetchResult) + return res.urlStr, res.imports, res.err +} + +type fetchResult struct { + urlStr string // e.g. "https://foo.com/x/bar?go-get=1" + imports []metaImport + err error +} + // metaImport represents the parsed <meta name="go-import" // content="prefix vcs reporoot" /> tags from HTML files. type metaImport struct { @@ -689,7 +835,10 @@ func expand(match map[string]string, s string) string { return s } -// vcsPaths lists the known vcs paths. +// vcsPaths defines the meaning of import paths referring to +// commonly-used VCS hosting sites (github.com/user/dir) +// and import paths referring to a fully-qualified importPath +// containing a VCS type (foo.com/repo.git/dir) var vcsPaths = []*vcsPath{ // Google Code - new syntax { @@ -722,15 +871,6 @@ var vcsPaths = []*vcsPath{ check: bitbucketVCS, }, - // Launchpad - { - prefix: "launchpad.net/", - re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - vcs: "bzr", - repo: "https://{root}", - check: launchpadVCS, - }, - // IBM DevOps Services (JazzHub) { prefix: "hub.jazz.net/git", @@ -740,13 +880,37 @@ var vcsPaths = []*vcsPath{ check: noVCSSuffix, }, + // Git at Apache + { + prefix: "git.apache.org", + re: `^(?P<root>git.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, + vcs: "git", + repo: "https://{root}", + }, + // General syntax for any server. + // Must be last. { re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`, ping: true, }, } +// vcsPathsAfterDynamic gives additional vcsPaths entries +// to try after the dynamic HTML check. +// This gives those sites a chance to introduce <meta> tags +// as part of a graceful transition away from the hard-coded logic. +var vcsPathsAfterDynamic = []*vcsPath{ + // Launchpad. See golang.org/issue/11436. + { + prefix: "launchpad.net/", + re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + vcs: "bzr", + repo: "https://{root}", + check: launchpadVCS, + }, +} + func init() { // fill in cached regexps. // Doing this eagerly discovers invalid regexp syntax @@ -754,6 +918,9 @@ func init() { for _, srv := range vcsPaths { srv.regexp = regexp.MustCompile(srv.re) } + for _, srv := range vcsPathsAfterDynamic { + srv.regexp = regexp.MustCompile(srv.re) + } } // noVCSSuffix checks that the repository name does not @@ -821,10 +988,25 @@ func bitbucketVCS(match map[string]string) error { url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}") data, err := httpGET(url) if err != nil { - return err - } - if err := json.Unmarshal(data, &resp); err != nil { - return fmt.Errorf("decoding %s: %v", url, err) + if httpErr, ok := err.(*httpError); ok && httpErr.statusCode == 403 { + // this may be a private repository. If so, attempt to determine which + // VCS it uses. See issue 5375. + root := match["root"] + for _, vcs := range []string{"git", "hg"} { + if vcsByCmd(vcs).ping("https", root) == nil { + resp.SCM = vcs + break + } + } + } + + if resp.SCM == "" { + return err + } + } else { + if err := json.Unmarshal(data, &resp); err != nil { + return fmt.Errorf("decoding %s: %v", url, err) + } } if vcsByCmd(resp.SCM) != nil { diff --git a/libgo/go/cmd/go/vcs_test.go b/libgo/go/cmd/go/vcs_test.go index 14d681ba6af..f5d5e4f4f0b 100644 --- a/libgo/go/cmd/go/vcs_test.go +++ b/libgo/go/cmd/go/vcs_test.go @@ -5,20 +5,15 @@ package main import ( - "runtime" + "internal/testenv" "testing" ) // Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath. // TODO(cmang): Add tests for SVN and BZR. func TestRepoRootForImportPath(t *testing.T) { - if testing.Short() { - t.Skip("skipping test to avoid external network") - } - switch runtime.GOOS { - case "nacl", "android": - t.Skipf("no networking available on %s", runtime.GOOS) - } + testenv.MustHaveExternalNetwork(t) + tests := []struct { path string want *repoRoot @@ -101,10 +96,34 @@ func TestRepoRootForImportPath(t *testing.T) { "hub.jazz.net/git/USER/pkgname", nil, }, + // Spaces are not valid in package name + { + "git.apache.org/package name/path/to/lib", + nil, + }, + // Should have ".git" suffix + { + "git.apache.org/package-name/path/to/lib", + nil, + }, + { + "git.apache.org/package-name.git", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.apache.org/package-name.git", + }, + }, + { + "git.apache.org/package-name_2.x.git/path/to/lib", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.apache.org/package-name_2.x.git", + }, + }, } for _, test := range tests { - got, err := repoRootForImportPath(test.path) + got, err := repoRootForImportPath(test.path, secure) want := test.want if want == nil { @@ -122,3 +141,35 @@ func TestRepoRootForImportPath(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + vcs *vcsCmd + url string + secure bool + }{ + {vcsGit, "http://example.com/foo.git", false}, + {vcsGit, "https://example.com/foo.git", true}, + {vcsBzr, "http://example.com/foo.bzr", false}, + {vcsBzr, "https://example.com/foo.bzr", true}, + {vcsSvn, "http://example.com/svn", false}, + {vcsSvn, "https://example.com/svn", true}, + {vcsHg, "http://example.com/foo.hg", false}, + {vcsHg, "https://example.com/foo.hg", true}, + {vcsGit, "ssh://user@example.com/foo.git", true}, + {vcsGit, "user@server:path/to/repo.git", false}, + {vcsGit, "user@server:", false}, + {vcsGit, "server:repo.git", false}, + {vcsGit, "server:path/to/repo.git", false}, + {vcsGit, "example.com:path/to/repo.git", false}, + {vcsGit, "path/that/contains/a:colon/repo.git", false}, + {vcsHg, "ssh://user@example.com/path/to/repo.hg", true}, + } + + for _, test := range tests { + secure := test.vcs.isSecure(test.url) + if secure != test.secure { + t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) + } + } +} diff --git a/libgo/go/cmd/go/vendor_test.go b/libgo/go/cmd/go/vendor_test.go new file mode 100644 index 00000000000..1e8cf9c8d26 --- /dev/null +++ b/libgo/go/cmd/go/vendor_test.go @@ -0,0 +1,258 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for vendoring semantics. + +package main_test + +import ( + "bytes" + "fmt" + "internal/testenv" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestVendorImports(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("list", "-f", "{{.ImportPath}} {{.Imports}}", "vend/...") + want := ` + vend [vend/vendor/p r] + vend/hello [fmt vend/vendor/strings] + vend/subdir [vend/vendor/p r] + vend/vendor/p [] + vend/vendor/q [] + vend/vendor/strings [] + vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r] + vend/x/invalid [vend/x/invalid/vendor/foo] + vend/x/vendor/p [] + vend/x/vendor/p/p [notfound] + vend/x/vendor/r [] + ` + want = strings.Replace(want+"\t", "\n\t\t", "\n", -1) + want = strings.TrimPrefix(want, "\n") + + have := tg.stdout.String() + + if have != want { + t.Errorf("incorrect go list output:\n%s", diffSortedOutputs(have, want)) + } +} + +func TestVendorRun(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.cd(filepath.Join(tg.pwd(), "testdata/src/vend/hello")) + tg.run("run", "hello.go") + tg.grepStdout("hello, world", "missing hello world output") +} + +func TestVendorGOPATH(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + changeVolume := func(s string, f func(s string) string) string { + vol := filepath.VolumeName(s) + return f(vol) + s[len(vol):] + } + gopath := changeVolume(filepath.Join(tg.pwd(), "testdata"), strings.ToLower) + tg.setenv("GOPATH", gopath) + tg.setenv("GO15VENDOREXPERIMENT", "1") + cd := changeVolume(filepath.Join(tg.pwd(), "testdata/src/vend/hello"), strings.ToUpper) + tg.cd(cd) + tg.run("run", "hello.go") + tg.grepStdout("hello, world", "missing hello world output") +} + +func TestVendorTest(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.cd(filepath.Join(tg.pwd(), "testdata/src/vend/hello")) + tg.run("test", "-v") + tg.grepStdout("TestMsgInternal", "missing use in internal test") + tg.grepStdout("TestMsgExternal", "missing use in external test") +} + +func TestVendorInvalid(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + + tg.runFail("build", "vend/x/invalid") + tg.grepStderr("must be imported as foo", "missing vendor import error") +} + +func TestVendorImportError(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + + tg.runFail("build", "vend/x/vendor/p/p") + + re := regexp.MustCompile(`cannot find package "notfound" in any of: + .*[\\/]testdata[\\/]src[\\/]vend[\\/]x[\\/]vendor[\\/]notfound \(vendor tree\) + .*[\\/]testdata[\\/]src[\\/]vend[\\/]vendor[\\/]notfound \(vendor tree\) + .*[\\/]src[\\/]notfound \(from \$GOROOT\) + .*[\\/]testdata[\\/]src[\\/]notfound \(from \$GOPATH\)`) + + if !re.MatchString(tg.stderr.String()) { + t.Errorf("did not find expected search list in error text") + } +} + +// diffSortedOutput prepares a diff of the already sorted outputs haveText and wantText. +// The diff shows common lines prefixed by a tab, lines present only in haveText +// prefixed by "unexpected: ", and lines present only in wantText prefixed by "missing: ". +func diffSortedOutputs(haveText, wantText string) string { + var diff bytes.Buffer + have := splitLines(haveText) + want := splitLines(wantText) + for len(have) > 0 || len(want) > 0 { + if len(want) == 0 || len(have) > 0 && have[0] < want[0] { + fmt.Fprintf(&diff, "unexpected: %s\n", have[0]) + have = have[1:] + continue + } + if len(have) == 0 || len(want) > 0 && want[0] < have[0] { + fmt.Fprintf(&diff, "missing: %s\n", want[0]) + want = want[1:] + continue + } + fmt.Fprintf(&diff, "\t%s\n", want[0]) + want = want[1:] + have = have[1:] + } + return diff.String() +} + +func splitLines(s string) []string { + x := strings.Split(s, "\n") + if x[len(x)-1] == "" { + x = x[:len(x)-1] + } + return x +} + +func TestVendorGet(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/v/m.go", ` + package main + import ("fmt"; "vendor.org/p") + func main() { + fmt.Println(p.C) + }`) + tg.tempFile("src/v/m_test.go", ` + package main + import ("fmt"; "testing"; "vendor.org/p") + func TestNothing(t *testing.T) { + fmt.Println(p.C) + }`) + tg.tempFile("src/v/vendor/vendor.org/p/p.go", ` + package p + const C = 1`) + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.cd(tg.path("src/v")) + tg.run("run", "m.go") + tg.run("test") + tg.run("list", "-f", "{{.Imports}}") + tg.grepStdout("v/vendor/vendor.org/p", "import not in vendor directory") + tg.run("list", "-f", "{{.TestImports}}") + tg.grepStdout("v/vendor/vendor.org/p", "test import not in vendor directory") + tg.run("get") + tg.run("get", "-t") +} + +func TestVendorGetUpdate(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("get", "github.com/rsc/go-get-issue-11864") + tg.run("get", "-u", "github.com/rsc/go-get-issue-11864") +} + +func TestVendorCache(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/testvendor")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.runFail("build", "p") + tg.grepStderr("must be imported as x", "did not fail to build p") +} + +func TestVendorTest2(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("get", "github.com/rsc/go-get-issue-11864") + + // build -i should work + tg.run("build", "-i", "github.com/rsc/go-get-issue-11864") + tg.run("build", "-i", "github.com/rsc/go-get-issue-11864/t") + + // test -i should work like build -i (golang.org/issue/11988) + tg.run("test", "-i", "github.com/rsc/go-get-issue-11864") + tg.run("test", "-i", "github.com/rsc/go-get-issue-11864/t") + + // test should work too + tg.run("test", "github.com/rsc/go-get-issue-11864") + tg.run("test", "github.com/rsc/go-get-issue-11864/t") + + // external tests should observe internal test exports (golang.org/issue/11977) + tg.run("test", "github.com/rsc/go-get-issue-11864/vendor/vendor.org/tx2") +} + +func TestVendorList(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("get", "github.com/rsc/go-get-issue-11864") + + tg.run("list", "-f", `{{join .TestImports "\n"}}`, "github.com/rsc/go-get-issue-11864/t") + tg.grepStdout("go-get-issue-11864/vendor/vendor.org/p", "did not find vendor-expanded p") + + tg.run("list", "-f", `{{join .XTestImports "\n"}}`, "github.com/rsc/go-get-issue-11864/tx") + tg.grepStdout("go-get-issue-11864/vendor/vendor.org/p", "did not find vendor-expanded p") + + tg.run("list", "-f", `{{join .XTestImports "\n"}}`, "github.com/rsc/go-get-issue-11864/vendor/vendor.org/tx2") + tg.grepStdout("go-get-issue-11864/vendor/vendor.org/tx2", "did not find vendor-expanded tx2") + + tg.run("list", "-f", `{{join .XTestImports "\n"}}`, "github.com/rsc/go-get-issue-11864/vendor/vendor.org/tx3") + tg.grepStdout("go-get-issue-11864/vendor/vendor.org/tx3", "did not find vendor-expanded tx3") +} + +func TestVendor12156(t *testing.T) { + // Former index out of range panic. + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/testvendor2")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.cd(filepath.Join(tg.pwd(), "testdata/testvendor2/src/p")) + tg.runFail("build", "p.go") + tg.grepStderrNot("panic", "panicked") + tg.grepStderr(`cannot find package "x"`, "wrong error") +} diff --git a/libgo/go/cmd/go/vet.go b/libgo/go/cmd/go/vet.go index 02ff54b2ac8..81b978e8dab 100644 --- a/libgo/go/cmd/go/vet.go +++ b/libgo/go/cmd/go/vet.go @@ -7,17 +7,17 @@ package main import "path/filepath" func init() { - addBuildFlagsNX(cmdVet) + addBuildFlags(cmdVet) } var cmdVet = &Command{ Run: runVet, - UsageLine: "vet [-n] [-x] [packages]", + UsageLine: "vet [-n] [-x] [build flags] [packages]", Short: "run go tool vet on packages", Long: ` Vet runs the Go vet command on the packages named by the import paths. -For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. +For more about vet, see 'go doc cmd/vet'. For more about specifying packages, see 'go help packages'. To run the vet tool with specific options, run 'go tool vet'. @@ -25,6 +25,8 @@ To run the vet tool with specific options, run 'go tool vet'. The -n flag prints commands that would be executed. The -x flag prints commands as they are executed. +For more about build flags, see 'go help build'. + See also: go fmt, go fix. `, } @@ -46,5 +48,5 @@ func runVetFiles(p *Package, files []string) { for i := range files { files[i] = filepath.Join(p.Dir, files[i]) } - run(tool("vet"), relPaths(files)) + run(buildToolExec, tool("vet"), relPaths(files)) } diff --git a/libgo/go/cmd/gofmt/doc.go b/libgo/go/cmd/gofmt/doc.go index 3fc0439548f..9d0cd328623 100644 --- a/libgo/go/cmd/gofmt/doc.go +++ b/libgo/go/cmd/gofmt/doc.go @@ -87,6 +87,13 @@ When invoked with -s gofmt will make the following source transformations where for x, _ = range v {...} will be simplified to: for x = range v {...} + + A range of the form: + for _ = range v {...} + will be simplified to: + for range v {...} + +This may result in changes that are incompatible with earlier versions of Go. */ package main diff --git a/libgo/go/cmd/gofmt/gofmt.go b/libgo/go/cmd/gofmt/gofmt.go index 81da21ff109..b2805ac05fb 100644 --- a/libgo/go/cmd/gofmt/gofmt.go +++ b/libgo/go/cmd/gofmt/gofmt.go @@ -13,6 +13,7 @@ import ( "go/printer" "go/scanner" "go/token" + "internal/format" "io" "io/ioutil" "os" @@ -87,7 +88,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error return err } - file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin) + file, sourceAdj, indentAdj, err := format.Parse(fileSet, filename, src, stdin) if err != nil { return err } @@ -106,7 +107,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error simplify(file) } - res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth}) + res, err := format.Format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth}) if err != nil { return err } @@ -234,154 +235,3 @@ func diff(b1, b2 []byte) (data []byte, err error) { return } - -// ---------------------------------------------------------------------------- -// Support functions -// -// The functions parse, format, and isSpace below are identical to the -// respective functions in src/go/format/format.go - keep them in sync! -// -// TODO(gri) Factor out this functionality, eventually. - -// parse parses src, which was read from the named file, -// as a Go source file, declaration, or statement list. -func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( - file *ast.File, - sourceAdj func(src []byte, indent int) []byte, - indentAdj int, - err error, -) { - // Try as whole source file. - file, err = parser.ParseFile(fset, filename, src, parserMode) - // If there's no error, return. If the error is that the source file didn't begin with a - // package line and source fragments are ok, fall through to - // try as a source fragment. Stop and return on any other error. - if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") { - return - } - - // If this is a declaration list, make it a source file - // by inserting a package clause. - // Insert using a ;, not a newline, so that the line numbers - // in psrc match the ones in src. - psrc := append([]byte("package p;"), src...) - file, err = parser.ParseFile(fset, filename, psrc, parserMode) - if err == nil { - sourceAdj = func(src []byte, indent int) []byte { - // Remove the package clause. - // Gofmt has turned the ; into a \n. - src = src[indent+len("package p\n"):] - return bytes.TrimSpace(src) - } - return - } - // If the error is that the source file didn't begin with a - // declaration, fall through to try as a statement list. - // Stop and return on any other error. - if !strings.Contains(err.Error(), "expected declaration") { - return - } - - // If this is a statement list, make it a source file - // by inserting a package clause and turning the list - // into a function body. This handles expressions too. - // Insert using a ;, not a newline, so that the line numbers - // in fsrc match the ones in src. - fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '}') - file, err = parser.ParseFile(fset, filename, fsrc, parserMode) - if err == nil { - sourceAdj = func(src []byte, indent int) []byte { - // Cap adjusted indent to zero. - if indent < 0 { - indent = 0 - } - // Remove the wrapping. - // Gofmt has turned the ; into a \n\n. - // There will be two non-blank lines with indent, hence 2*indent. - src = src[2*indent+len("package p\n\nfunc _() {"):] - src = src[:len(src)-(indent+len("\n}\n"))] - return bytes.TrimSpace(src) - } - // Gofmt has also indented the function body one level. - // Adjust that with indentAdj. - indentAdj = -1 - } - - // Succeeded, or out of options. - return -} - -// format formats the given package file originally obtained from src -// and adjusts the result based on the original source via sourceAdj -// and indentAdj. -func format( - fset *token.FileSet, - file *ast.File, - sourceAdj func(src []byte, indent int) []byte, - indentAdj int, - src []byte, - cfg printer.Config, -) ([]byte, error) { - if sourceAdj == nil { - // Complete source file. - var buf bytes.Buffer - err := cfg.Fprint(&buf, fset, file) - if err != nil { - return nil, err - } - return buf.Bytes(), nil - } - - // Partial source file. - // Determine and prepend leading space. - i, j := 0, 0 - for j < len(src) && isSpace(src[j]) { - if src[j] == '\n' { - i = j + 1 // byte offset of last line in leading space - } - j++ - } - var res []byte - res = append(res, src[:i]...) - - // Determine and prepend indentation of first code line. - // Spaces are ignored unless there are no tabs, - // in which case spaces count as one tab. - indent := 0 - hasSpace := false - for _, b := range src[i:j] { - switch b { - case ' ': - hasSpace = true - case '\t': - indent++ - } - } - if indent == 0 && hasSpace { - indent = 1 - } - for i := 0; i < indent; i++ { - res = append(res, '\t') - } - - // Format the source. - // Write it without any leading and trailing space. - cfg.Indent = indent + indentAdj - var buf bytes.Buffer - err := cfg.Fprint(&buf, fset, file) - if err != nil { - return nil, err - } - res = append(res, sourceAdj(buf.Bytes(), cfg.Indent)...) - - // Determine and append trailing space. - i = len(src) - for i > 0 && isSpace(src[i-1]) { - i-- - } - return append(res, src[i:]...), nil -} - -func isSpace(b byte) bool { - return b == ' ' || b == '\t' || b == '\n' || b == '\r' -} diff --git a/libgo/go/cmd/gofmt/long_test.go b/libgo/go/cmd/gofmt/long_test.go index 237b86021bf..df9a878df44 100644 --- a/libgo/go/cmd/gofmt/long_test.go +++ b/libgo/go/cmd/gofmt/long_test.go @@ -15,6 +15,7 @@ import ( "go/ast" "go/printer" "go/token" + "internal/format" "io" "os" "path/filepath" @@ -32,7 +33,7 @@ var ( ) func gofmt(fset *token.FileSet, filename string, src *bytes.Buffer) error { - f, _, _, err := parse(fset, filename, src.Bytes(), false) + f, _, _, err := format.Parse(fset, filename, src.Bytes(), false) if err != nil { return err } @@ -60,7 +61,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) { // exclude files w/ syntax errors (typically test cases) fset := token.NewFileSet() - if _, _, _, err = parse(fset, filename, b1.Bytes(), false); err != nil { + if _, _, _, err = format.Parse(fset, filename, b1.Bytes(), false); err != nil { if *verbose { fmt.Fprintf(os.Stderr, "ignoring %s\n", err) } diff --git a/libgo/go/cmd/gofmt/rewrite.go b/libgo/go/cmd/gofmt/rewrite.go index d267cfcc1dc..069f96622ca 100644 --- a/libgo/go/cmd/gofmt/rewrite.go +++ b/libgo/go/cmd/gofmt/rewrite.go @@ -154,7 +154,7 @@ func isWildcard(s string) bool { return size == len(s) && unicode.IsLower(rune) } -// match returns true if pattern matches val, +// match reports whether pattern matches val, // recording wildcard submatches in m. // If m == nil, match checks whether pattern == val. func match(m map[string]reflect.Value, pattern, val reflect.Value) bool { |