From 507a88c39bb1089b9d44facb7dd3449a9b5a3e10 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 1 Oct 2020 13:37:06 -0400 Subject: cmd/go/internal/modfetch: always extract module directories in place Previously by default, we extracted modules to a temporary directory, then renamed it into place. This failed with ERROR_ACCESS_DENIED on Windows if another process (usually an anti-virus scanner) opened files in the temporary directory. Since Go 1.15, users have been able to set GODEBUG=modcacheunzipinplace=1 to opt into new behavior: we extract modules at their final location, and we create and later delete a .partial file to prevent the directory from being used if we crash. .partial files are recognized by Go 1.14.2 and later. With this change, the new behavior is the only behavior. modcacheunzipinplace is no longer recognized. Fixes #36568 Change-Id: Iff19fca5cd6eaa3597975a69fa05c4cb1b834bd6 Reviewed-on: https://go-review.googlesource.com/c/go/+/258798 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Trust: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modfetch/fetch.go | 105 ++++++++++------------------------ 1 file changed, 31 insertions(+), 74 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 01d8f007ac..1d90002faa 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -63,12 +63,9 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) defer span.Done() - // If the directory exists, and no .partial file exists, the module has - // already been completely extracted. .partial files may be created when a - // module zip directory is extracted in place instead of being extracted to a - // temporary directory and renamed. dir, err = DownloadDir(mod) if err == nil { + // The directory has already been completely extracted (no .partial file exists). return dir, nil } else if dir == "" || !errors.Is(err, os.ErrNotExist) { return "", err @@ -88,6 +85,9 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { } defer unlock() + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) + defer span.Done() + // Check whether the directory was populated while we were waiting on the lock. _, dirErr := DownloadDir(mod) if dirErr == nil { @@ -95,10 +95,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { } _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs, as well - // as partially extracted diectories created by future versions of cmd/go. - // This is only safe to do because the lock file ensures that their writers - // are no longer active. + // Clean up any remaining temporary directories created by old versions + // (before 1.16), as well as partially extracted directories (indicated by + // DownloadDirPartialError, usually because of a .partial file). This is only + // safe to do because the lock file ensures that their writers are no longer + // active. parentDir := filepath.Dir(dir) tmpPrefix := filepath.Base(dir) + ".tmp-" if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil { @@ -116,88 +117,44 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { if err != nil { return "", err } - if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { - return "", err - } - // Extract the module zip directory. + // Extract the module zip directory at its final location. // - // By default, we extract to a temporary directory, then atomically rename to - // its final location. We use the existence of the source directory to signal - // that it has been extracted successfully (see DownloadDir). If someone - // deletes the entire directory (e.g., as an attempt to prune out file - // corruption), the module cache will still be left in a recoverable - // state. + // To prevent other processes from reading the directory if we crash, + // create a .partial file before extracting the directory, and delete + // the .partial file afterward (all while holding the lock). // - // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if - // another process opens files in the temporary directory. This is partially - // mitigated by using robustio.Rename, which retries os.Rename for a short - // time. + // Before Go 1.16, we extracted to a temporary directory with a random name + // then renamed it into place with os.Rename. On Windows, this failed with + // ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner) + // opened files in the temporary directory. // - // To avoid this error completely, if unzipInPlace is set, we instead create a - // .partial file (indicating the directory isn't fully extracted), then we - // extract the directory at its final location, then we delete the .partial - // file. This is not the default behavior because older versions of Go may - // simply stat the directory to check whether it exists without looking for a - // .partial file. If multiple versions run concurrently, the older version may - // assume a partially extracted directory is complete. - // TODO(golang.org/issue/36568): when these older versions are no longer - // supported, remove the old default behavior and the unzipInPlace flag. + // Go 1.14.2 and higher respect .partial files. Older versions may use + // partially extracted directories. 'go mod verify' can detect this, + // and 'go clean -modcache' can fix it. if err := os.MkdirAll(parentDir, 0777); err != nil { return "", err } - - ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) - if unzipInPlace { - if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { - return "", err - } - if err := modzip.Unzip(dir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - if rmErr := RemoveAll(dir); rmErr == nil { - os.Remove(partialPath) - } - return "", err - } - if err := os.Remove(partialPath); err != nil { - return "", err - } - } else { - tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) - if err != nil { - return "", err - } - if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - RemoveAll(tmpDir) - return "", err - } - if err := robustio.Rename(tmpDir, dir); err != nil { - RemoveAll(tmpDir) - return "", err + if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { + return "", err + } + if err := modzip.Unzip(dir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) + if rmErr := RemoveAll(dir); rmErr == nil { + os.Remove(partialPath) } + return "", err + } + if err := os.Remove(partialPath); err != nil { + return "", err } - defer span.Done() if !cfg.ModCacheRW { - // Make dir read-only only *after* renaming it. - // os.Rename was observed to fail for read-only directories on macOS. makeDirsReadOnly(dir) } return dir, nil } -var unzipInPlace bool - -func init() { - for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { - if f == "modcacheunzipinplace=1" { - unzipInPlace = true - break - } - } -} - var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the -- cgit v1.2.1 From fe2cfb74ba6352990f5b41260b99e80f78e4a90a Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 1 Oct 2020 14:49:33 -0700 Subject: all: drop 387 support My last 387 CL. So sad ... ... ... ... not! Fixes #40255 Change-Id: I8d4ddb744b234b8adc735db2f7c3c7b6d8bbdfa4 Reviewed-on: https://go-review.googlesource.com/c/go/+/258957 Trust: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- src/cmd/go/internal/cfg/cfg.go | 3 --- src/cmd/go/internal/envcmd/env.go | 5 ++++- src/cmd/go/internal/help/helpdoc.go | 3 --- src/cmd/go/internal/work/exec.go | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 9bf1db73ef..ebbaf04115 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -244,7 +244,6 @@ var ( // Used in envcmd.MkEnv and build ID computations. GOARM = envOr("GOARM", fmt.Sprint(objabi.GOARM)) - GO386 = envOr("GO386", objabi.GO386) GOMIPS = envOr("GOMIPS", objabi.GOMIPS) GOMIPS64 = envOr("GOMIPS64", objabi.GOMIPS64) GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64)) @@ -268,8 +267,6 @@ func GetArchEnv() (key, val string) { switch Goarch { case "arm": return "GOARM", GOARM - case "386": - return "GO386", GO386 case "mips", "mipsle": return "GOMIPS", GOMIPS case "mips64", "mips64le": diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 7bd75f7305..ee0bb0d0b2 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -497,7 +497,10 @@ func lineToKey(line string) string { } // sortKeyValues sorts a sequence of lines by key. -// It differs from sort.Strings in that GO386= sorts after GO=. +// It differs from sort.Strings in that keys which are GOx where x is an ASCII +// character smaller than = sort after GO=. +// (There are no such keys currently. It used to matter for GO386 which was +// removed in Go 1.16.) func sortKeyValues(lines []string) { sort.Slice(lines, func(i, j int) bool { return lineToKey(lines[i]) < lineToKey(lines[j]) diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 0ae5fd7ca9..befa10a0e4 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -581,9 +581,6 @@ Architecture-specific environment variables: GOARM For GOARCH=arm, the ARM architecture for which to compile. Valid values are 5, 6, 7. - GO386 - For GOARCH=386, the floating point instruction set. - Valid values are 387, sse2. GOMIPS For GOARCH=mips{,le}, whether to use floating point instructions. Valid values are hardfloat (default), softfloat. diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 51fc2b588d..e68b322c7d 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -271,7 +271,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) @@ -1175,7 +1175,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) -- cgit v1.2.1 From a65bc048bf388e399af9bcfd726cd0f11bba7c8e Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 2 Oct 2020 16:17:30 -0700 Subject: cmd/go: use cmd/internal/pkgpath for gccgo pkgpath symbol Fixes #37272 Change-Id: I6554fd5e5400acb20c5a7e96b1d6cb1a1afb9871 Reviewed-on: https://go-review.googlesource.com/c/go/+/259299 Trust: Ian Lance Taylor Run-TryBot: Ian Lance Taylor TryBot-Result: Go Bot Reviewed-by: Than McIntosh --- src/cmd/go/internal/work/gccgo.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go index 4c1f36dbd6..dd5adf2d7b 100644 --- a/src/cmd/go/internal/work/gccgo.go +++ b/src/cmd/go/internal/work/gccgo.go @@ -11,11 +11,13 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/str" + "cmd/internal/pkgpath" ) // The Gccgo toolchain. @@ -174,7 +176,7 @@ func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]strin ofiles = append(ofiles, ofile) sfile = mkAbs(p.Dir, sfile) defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) } defs = tools.maybePIC(defs) @@ -531,7 +533,7 @@ func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error cfile = mkAbs(p.Dir, cfile) defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} defs = append(defs, b.gccArchArgs()...) - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) } compiler := envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) @@ -568,14 +570,19 @@ func gccgoPkgpath(p *load.Package) string { return p.ImportPath } -func gccgoCleanPkgpath(p *load.Package) string { - clean := func(r rune) rune { - switch { - case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', - '0' <= r && r <= '9': - return r +var gccgoToSymbolFuncOnce sync.Once +var gccgoToSymbolFunc func(string) string + +func (tools gccgoToolchain) gccgoCleanPkgpath(b *Builder, p *load.Package) string { + gccgoToSymbolFuncOnce.Do(func() { + fn, err := pkgpath.ToSymbolFunc(tools.compiler(), b.WorkDir) + if err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() } - return '_' - } - return strings.Map(clean, gccgoPkgpath(p)) + gccgoToSymbolFunc = fn + }) + + return gccgoToSymbolFunc(gccgoPkgpath(p)) } -- cgit v1.2.1 From ab2a5b48665eed6d670d719cdef5335bc3602359 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 11 Aug 2020 12:57:01 -0400 Subject: cmd/go: add basic support for overlays This CL adds basic support for listing packages with overlays. The new cmd/go/internal/fs package adds an abstraction for communicating with the file system that will open files according to their overlaid paths, and provides functions to override those in the build context to open overlaid files. There is also some support for executing builds on packages with overlays. In cmd/go/internal/work.(*Builder).build, paths are mapped to their overlaid paths before they are given as arguments to tools. For #39958 Change-Id: I5ec0eb9ebbca303e2f1e7dbe22ec32613bc1fd17 Reviewed-on: https://go-review.googlesource.com/c/go/+/253747 Trust: Michael Matloob Trust: Jay Conrod Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/cfg/cfg.go | 12 + src/cmd/go/internal/envcmd/env.go | 5 + src/cmd/go/internal/fsys/fsys.go | 426 ++++++++++++++++++++++++++++++ src/cmd/go/internal/fsys/fsys_test.go | 479 ++++++++++++++++++++++++++++++++++ src/cmd/go/internal/imports/scan.go | 7 +- src/cmd/go/internal/modload/import.go | 51 +--- src/cmd/go/internal/modload/init.go | 5 + src/cmd/go/internal/search/search.go | 1 + src/cmd/go/internal/work/build.go | 3 + src/cmd/go/internal/work/gc.go | 23 +- src/cmd/go/internal/work/init.go | 4 + 11 files changed, 963 insertions(+), 53 deletions(-) create mode 100644 src/cmd/go/internal/fsys/fsys.go create mode 100644 src/cmd/go/internal/fsys/fsys_test.go (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index ebbaf04115..9169c12d8f 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -11,6 +11,7 @@ import ( "fmt" "go/build" "internal/cfg" + "io" "io/ioutil" "os" "path/filepath" @@ -18,6 +19,8 @@ import ( "strings" "sync" + "cmd/go/internal/fsys" + "cmd/internal/objabi" ) @@ -104,6 +107,15 @@ func defaultContext() build.Context { // Nothing to do here. } + ctxt.OpenFile = func(path string) (io.ReadCloser, error) { + return fsys.Open(path) + } + ctxt.ReadDir = fsys.ReadDir + ctxt.IsDir = func(path string) bool { + isDir, err := fsys.IsDir(path) + return err == nil && isDir + } + return ctxt } diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index ee0bb0d0b2..e1f2400f60 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -21,6 +21,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cache" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" @@ -197,6 +198,10 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { env := cfg.CmdEnv env = append(env, ExtraEnvVars()...) + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Do we need to call ExtraEnvVarsCostly, which is a bit expensive? // Only if we're listing all environment variables ("go env") // or the variables being requested are in the extra list. diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go new file mode 100644 index 0000000000..d64ce0aba1 --- /dev/null +++ b/src/cmd/go/internal/fsys/fsys.go @@ -0,0 +1,426 @@ +// Package fsys is an abstraction for reading files that +// allows for virtual overlays on top of the files on disk. +package fsys + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// OverlayFile is the path to a text file in the OverlayJSON format. +// It is the value of the -overlay flag. +var OverlayFile string + +// OverlayJSON is the format overlay files are expected to be in. +// The Replace map maps from overlaid paths to replacement paths: +// the Go command will forward all reads trying to open +// each overlaid path to its replacement path, or consider the overlaid +// path not to exist if the replacement path is empty. +type OverlayJSON struct { + Replace map[string]string +} + +type node struct { + actualFilePath string // empty if a directory + children map[string]*node // path element → file or directory +} + +func (n *node) isDir() bool { + return n.actualFilePath == "" && n.children != nil +} + +func (n *node) isDeleted() bool { + return n.actualFilePath == "" && n.children == nil +} + +// TODO(matloob): encapsulate these in an io/fs-like interface +var overlay map[string]*node // path -> file or directory node +var cwd string // copy of base.Cwd to avoid dependency + +// Canonicalize a path for looking it up in the overlay. +// Important: filepath.Join(cwd, path) doesn't always produce +// the correct absolute path if path is relative, because on +// Windows producing the correct absolute path requires making +// a syscall. So this should only be used when looking up paths +// in the overlay, or canonicalizing the paths in the overlay. +func canonicalize(path string) string { + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { + // On Windows filepath.Join(cwd, path) doesn't always work. In general + // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go + // use filepath.Join(cwd, path), but cmd/go specifically supports Windows + // paths that start with "\" which implies the path is relative to the + // volume of the working directory. See golang.org/issue/8130. + return filepath.Join(v, path) + } + + // Make the path absolute. + return filepath.Join(cwd, path) +} + +// Init initializes the overlay, if one is being used. +func Init(wd string) error { + if overlay != nil { + // already initialized + return nil + } + + cwd = wd + + if OverlayFile == "" { + return nil + } + + b, err := ioutil.ReadFile(OverlayFile) + if err != nil { + return fmt.Errorf("reading overlay file: %v", err) + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(b, &overlayJSON); err != nil { + return fmt.Errorf("parsing overlay JSON: %v", err) + } + + return initFromJSON(overlayJSON) +} + +func initFromJSON(overlayJSON OverlayJSON) error { + // Canonicalize the paths in in the overlay map. + // Use reverseCanonicalized to check for collisions: + // no two 'from' paths should canonicalize to the same path. + overlay = make(map[string]*node) + reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates + // Build a table of file and directory nodes from the replacement map. + + // Remove any potential non-determinism from iterating over map by sorting it. + replaceFrom := make([]string, 0, len(overlayJSON.Replace)) + for k := range overlayJSON.Replace { + replaceFrom = append(replaceFrom, k) + } + sort.Strings(replaceFrom) + + for _, from := range replaceFrom { + to := overlayJSON.Replace[from] + // Canonicalize paths and check for a collision. + if from == "" { + return fmt.Errorf("empty string key in overlay file Replace map") + } + cfrom := canonicalize(from) + if to != "" { + // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". + to = canonicalize(to) + } + if otherFrom, seen := reverseCanonicalized[cfrom]; seen { + return fmt.Errorf( + "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) + } + reverseCanonicalized[cfrom] = from + from = cfrom + + // Create node for overlaid file. + dir, base := filepath.Dir(from), filepath.Base(from) + if n, ok := overlay[from]; ok { + // All 'from' paths in the overlay are file paths. Since the from paths + // are in a map, they are unique, so if the node already exists we added + // it below when we create parent directory nodes. That is, that + // both a file and a path to one of its parent directories exist as keys + // in the Replace map. + // + // This only applies if the overlay directory has any files or directories + // in it: placeholder directories that only contain deleted files don't + // count. They are safe to be overwritten with actual files. + for _, f := range n.children { + if !f.isDeleted() { + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) + } + } + } + overlay[from] = &node{actualFilePath: to} + + // Add parent directory nodes to overlay structure. + childNode := overlay[from] + for { + dirNode := overlay[dir] + if dirNode == nil || dirNode.isDeleted() { + dirNode = &node{children: make(map[string]*node)} + overlay[dir] = dirNode + } + if childNode.isDeleted() { + // Only create one parent for a deleted file: + // the directory only conditionally exists if + // there are any non-deleted children, so + // we don't create their parents. + if dirNode.isDir() { + dirNode.children[base] = childNode + } + break + } + if !dirNode.isDir() { + // This path already exists as a file, so it can't be a parent + // directory. See comment at error above. + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) + } + dirNode.children[base] = childNode + parent := filepath.Dir(dir) + if parent == dir { + break // reached the top; there is no parent + } + dir, base = parent, filepath.Base(dir) + childNode = dirNode + } + } + + return nil +} + +// IsDir returns true if path is a directory on disk or in the +// overlay. +func IsDir(path string) (bool, error) { + path = canonicalize(path) + + if _, ok := parentIsOverlayFile(path); ok { + return false, nil + } + + if n, ok := overlay[path]; ok { + return n.isDir(), nil + } + + fi, err := os.Stat(path) + if err != nil { + return false, err + } + + return fi.IsDir(), nil +} + +// parentIsOverlayFile returns whether name or any of +// its parents are directories in the overlay, and the first parent found, +// including name itself, that's a directory in the overlay. +func parentIsOverlayFile(name string) (string, bool) { + if overlay != nil { + // Check if name can't possibly be a directory because + // it or one of its parents is overlaid with a file. + // TODO(matloob): Maybe save this to avoid doing it every time? + prefix := name + for { + node := overlay[prefix] + if node != nil && !node.isDir() { + return prefix, true + } + parent := filepath.Dir(prefix) + if parent == prefix { + break + } + prefix = parent + } + } + + return "", false +} + +// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles +// that the argument is not a directory, so that IsDirWithGoFiles doesn't +// return an error. +var errNotDir = errors.New("not a directory") + +// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. +// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory +// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). +func readDir(dir string) ([]os.FileInfo, error) { + fis, err := ioutil.ReadDir(dir) + if err == nil { + return fis, nil + } + + if os.IsNotExist(err) { + return nil, err + } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } else { + return nil, err + } +} + +// ReadDir provides a slice of os.FileInfo entries corresponding +// to the overlaid files in the directory. +func ReadDir(dir string) ([]os.FileInfo, error) { + dir = canonicalize(dir) + if _, ok := parentIsOverlayFile(dir); ok { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + + dirNode := overlay[dir] + if dirNode == nil { + return readDir(dir) + } else if dirNode.isDeleted() { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: os.ErrNotExist} + } + diskfis, err := readDir(dir) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { + return nil, err + } + + // Stat files in overlay to make composite list of fileinfos + files := make(map[string]os.FileInfo) + for _, f := range diskfis { + files[f.Name()] = f + } + for name, to := range dirNode.children { + switch { + case to.isDir(): + files[name] = fakeDir(name) + case to.isDeleted(): + delete(files, name) + default: + // This is a regular file. + f, err := os.Lstat(to.actualFilePath) + if err != nil { + files[name] = missingFile(name) + continue + } else if f.IsDir() { + return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", + filepath.Join(dir, name), to.actualFilePath) + } + // Add a fileinfo for the overlaid file, so that it has + // the original file's name, but the overlaid file's metadata. + files[name] = fakeFile{name, f} + } + } + sortedFiles := diskfis[:0] + for _, f := range files { + sortedFiles = append(sortedFiles, f) + } + sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) + return sortedFiles, nil +} + +// OverlayPath returns the path to the overlaid contents of the +// file, the empty string if the overlay deletes the file, or path +// itself if the file is not in the overlay, the file is a directory +// in the overlay, or there is no overlay. +// It returns true if the path is overlaid with a regular file +// or deleted, and false otherwise. +func OverlayPath(path string) (string, bool) { + if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { + return p.actualFilePath, ok + } + + return path, false +} + +// Open opens the file at or overlaid on the given path. +func Open(path string) (*os.File, error) { + cpath := canonicalize(path) + if node, ok := overlay[cpath]; ok { + if node.isDir() { + return nil, &os.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} + } + return os.Open(node.actualFilePath) + } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + // The file is deleted explicitly in the Replace map, + // or implicitly because one of its parent directories was + // replaced by a file. + return nil, &os.PathError{ + Op: "Open", + Path: path, + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} + } else { + return os.Open(cpath) + } +} + +// IsDirWithGoFiles reports whether dir is a directory containing Go files +// either on disk or in the overlay. +func IsDirWithGoFiles(dir string) (bool, error) { + fis, err := ReadDir(dir) + if os.IsNotExist(err) || errors.Is(err, errNotDir) { + return false, nil + } else if err != nil { + return false, err + } + + var firstErr error + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // TODO(matloob): this enforces that the "from" in the map + // has a .go suffix, but the actual destination file + // doesn't need to have a .go suffix. Is this okay with the + // compiler? + if !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if fi.Mode().IsRegular() { + return true, nil + } + + // fi is the result of an Lstat, so it doesn't follow symlinks. + // But it's okay if the file is a symlink pointing to a regular + // file, so use os.Stat to follow symlinks and check that. + actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) + if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { + return true, nil + } else if err != nil && firstErr == nil { + firstErr = err + } + } + + // No go files found in directory. + return false, firstErr +} + +// fakeFile provides an os.FileInfo implementation for an overlaid file, +// so that the file has the name of the overlaid file, but takes all +// other characteristics of the replacement file. +type fakeFile struct { + name string + real os.FileInfo +} + +func (f fakeFile) Name() string { return f.name } +func (f fakeFile) Size() int64 { return f.real.Size() } +func (f fakeFile) Mode() os.FileMode { return f.real.Mode() } +func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } +func (f fakeFile) IsDir() bool { return f.real.IsDir() } +func (f fakeFile) Sys() interface{} { return f.real.Sys() } + +// missingFile provides an os.FileInfo for an overlaid file where the +// destination file in the overlay doesn't exist. It returns zero values +// for the fileInfo methods other than Name, set to the file's name, and Mode +// set to ModeIrregular. +type missingFile string + +func (f missingFile) Name() string { return string(f) } +func (f missingFile) Size() int64 { return 0 } +func (f missingFile) Mode() os.FileMode { return os.ModeIrregular } +func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } +func (f missingFile) IsDir() bool { return false } +func (f missingFile) Sys() interface{} { return nil } + +// fakeDir provides an os.FileInfo implementation for directories that are +// implicitly created by overlaid files. Each directory in the +// path of an overlaid file is considered to exist in the overlay filesystem. +type fakeDir string + +func (f fakeDir) Name() string { return string(f) } +func (f fakeDir) Size() int64 { return 0 } +func (f fakeDir) Mode() os.FileMode { return os.ModeDir | 0500 } +func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } +func (f fakeDir) IsDir() bool { return true } +func (f fakeDir) Sys() interface{} { return nil } diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go new file mode 100644 index 0000000000..4b53059427 --- /dev/null +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -0,0 +1,479 @@ +package fsys + +import ( + "cmd/go/internal/txtar" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +// initOverlay resets the overlay state to reflect the config. +// config should be a text archive string. The comment is the overlay config +// json, and the files, in the archive are laid out in a temp directory +// that cwd is set to. +func initOverlay(t *testing.T, config string) { + t.Helper() + + // Create a temporary directory and chdir to it. + prevwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + cwd = t.TempDir() + if err := os.Chdir(cwd); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + overlay = nil + if err := os.Chdir(prevwd); err != nil { + t.Fatal(err) + } + }) + + a := txtar.Parse([]byte(config)) + for _, f := range a.Files { + name := filepath.Join(cwd, f.Name) + if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(name, f.Data, 0666); err != nil { + t.Fatal(err) + } + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(a.Comment, &overlayJSON); err != nil { + t.Fatal(fmt.Errorf("parsing overlay JSON: %v", err)) + } + + initFromJSON(overlayJSON) +} + +func TestIsDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6": "" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- subdir6/file6.txt -- +six +`) + + testCases := []struct { + path string + want, wantErr bool + }{ + {"", true, true}, + {".", true, false}, + {cwd, true, false}, + {cwd + string(filepath.Separator), true, false}, + // subdir1 is only on disk + {filepath.Join(cwd, "subdir1"), true, false}, + {"subdir1", true, false}, + {"subdir1" + string(filepath.Separator), true, false}, + {"subdir1/file1.txt", false, false}, + {"subdir1/doesntexist.txt", false, true}, + {"doesntexist", false, true}, + // subdir2 is only in overlay + {filepath.Join(cwd, "subdir2"), true, false}, + {"subdir2", true, false}, + {"subdir2" + string(filepath.Separator), true, false}, + {"subdir2/file2.txt", false, false}, + {"subdir2/doesntexist.txt", false, true}, + // subdir3 has files on disk and in overlay + {filepath.Join(cwd, "subdir3"), true, false}, + {"subdir3", true, false}, + {"subdir3" + string(filepath.Separator), true, false}, + {"subdir3/file3a.txt", false, false}, + {"subdir3/file3b.txt", false, false}, + {"subdir3/doesntexist.txt", false, true}, + // subdir4 is overlaid with a file + {filepath.Join(cwd, "subdir4"), false, false}, + {"subdir4", false, false}, + {"subdir4" + string(filepath.Separator), false, false}, + {"subdir4/file4.txt", false, false}, + {"subdir4/doesntexist.txt", false, false}, + // subdir5 doesn't exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir5"), false, false}, + {"subdir5", false, false}, + {"subdir5" + string(filepath.Separator), false, false}, + {"subdir5/file5.txt", false, false}, + {"subdir5/doesntexist.txt", false, false}, + // subdir6 does exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir6"), false, false}, + {"subdir6", false, false}, + {"subdir6" + string(filepath.Separator), false, false}, + {"subdir6/file6.txt", false, false}, + {"subdir6/doesntexist.txt", false, false}, + } + + for _, tc := range testCases { + got, err := IsDir(tc.path) + if err != nil { + if !tc.wantErr { + t.Errorf("IsDir(%q): got error with string %q, want no error", tc.path, err.Error()) + } + continue + } + if tc.wantErr { + t.Errorf("IsDir(%q): got no error, want error", tc.path) + } + if tc.want != got { + t.Errorf("IsDir(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +func TestReadDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6/asubsubdir/afile.txt": "overlayfiles/subdir6_asubsubdir_afile.txt", + "subdir6/asubsubdir/zfile.txt": "overlayfiles/subdir6_asubsubdir_zfile.txt", + "subdir6/zsubsubdir/file.txt": "overlayfiles/subdir6_zsubsubdir_file.txt", + "subdir7/asubsubdir/file.txt": "overlayfiles/subdir7_asubsubdir_file.txt", + "subdir7/zsubsubdir/file.txt": "overlayfiles/subdir7_zsubsubdir_file.txt", + "subdir8/doesntexist": "this_file_doesnt_exist_anywhere", + "other/pointstodir": "overlayfiles/this_is_a_directory", + "parentoverwritten/subdir1": "overlayfiles/parentoverwritten_subdir1", + "subdir9/this_file_is_overlaid.txt": "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir10/only_deleted_file.txt": "", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "textfile.txt/file.go": "overlayfiles/textfile_txt_file.go" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- subdir6/file.txt -- +-- subdir6/asubsubdir/file.txt -- +-- subdir6/anothersubsubdir/file.txt -- +-- subdir9/this_file_is_overlaid.txt -- +-- subdir10/only_deleted_file.txt -- +this will be deleted in overlay +-- subdir11/deleted.txt -- +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- textfile.txt -- +this will be overridden by textfile.txt/file.go +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- overlayfiles/subdir6_asubsubdir_afile.txt -- +-- overlayfiles/subdir6_asubsubdir_zfile.txt -- +-- overlayfiles/subdir6_zsubsubdir_file.txt -- +-- overlayfiles/subdir7_asubsubdir_file.txt -- +-- overlayfiles/subdir7_zsubsubdir_file.txt -- +-- overlayfiles/parentoverwritten_subdir1 -- +x +-- overlayfiles/subdir9_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/subdir11 -- +-- overlayfiles/this_is_a_directory/file.txt -- +-- overlayfiles/textfile_txt_file.go -- +x +`) + + testCases := map[string][]struct { + name string + size int64 + isDir bool + }{ + ".": { + {"other", 0, true}, + {"overlayfiles", 0, true}, + {"parentoverwritten", 0, true}, + {"subdir1", 0, true}, + {"subdir10", 0, true}, + {"subdir11", 0, false}, + {"subdir2", 0, true}, + {"subdir3", 0, true}, + {"subdir4", 2, false}, + // no subdir5. + {"subdir6", 0, true}, + {"subdir7", 0, true}, + {"subdir8", 0, true}, + {"subdir9", 0, true}, + {"textfile.txt", 0, true}, + }, + "subdir1": {{"file1.txt", 1, false}}, + "subdir2": {{"file2.txt", 2, false}}, + "subdir3": {{"file3a.txt", 3, false}, {"file3b.txt", 6, false}}, + "subdir6": { + {"anothersubsubdir", 0, true}, + {"asubsubdir", 0, true}, + {"file.txt", 0, false}, + {"zsubsubdir", 0, true}, + }, + "subdir6/asubsubdir": {{"afile.txt", 0, false}, {"file.txt", 0, false}, {"zfile.txt", 0, false}}, + "subdir8": {{"doesntexist", 0, false}}, // entry is returned even if destination file doesn't exist + // check that read dir actually redirects files that already exist + // the original this_file_is_overlaid.txt is empty + "subdir9": {{"this_file_is_overlaid.txt", 9, false}}, + "subdir10": {}, + "parentoverwritten": {{"subdir1", 2, false}}, + "textfile.txt": {{"file.go", 2, false}}, + } + + for dir, want := range testCases { + fis, err := ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir(%q): got error %q, want no error", dir, err) + } + if len(fis) != len(want) { + t.Fatalf("ReadDir(%q) result: got %v entries; want %v entries", dir, len(fis), len(want)) + } + for i := range fis { + if fis[i].Name() != want[i].name { + t.Fatalf("ReadDir(%q) entry %v: got Name() = %v, want %v", dir, i, fis[i].Name(), want[i].name) + } + if fis[i].IsDir() != want[i].isDir { + t.Fatalf("ReadDir(%q) entry %v: got IsDir() = %v, want %v", dir, i, fis[i].IsDir(), want[i].isDir) + } + if want[i].isDir { + // We don't try to get size right for directories + continue + } + if fis[i].Size() != want[i].size { + t.Fatalf("ReadDir(%q) entry %v: got Size() = %v, want %v", dir, i, fis[i].Size(), want[i].size) + } + } + } + + errCases := []string{ + "subdir1/file1.txt", // regular file on disk + "subdir2/file2.txt", // regular file in overlay + "subdir4", // directory overlaid with regular file + "subdir5", // directory deleted in overlay + "parentoverwritten/subdir1/subdir2/subdir3", // parentoverwritten/subdir1 overlaid with regular file + "parentoverwritten/subdir1/subdir2", // parentoverwritten/subdir1 overlaid with regular file + "subdir11", // directory with deleted child, overlaid with regular file + "other/pointstodir", + } + + for _, dir := range errCases { + _, gotErr := ReadDir(dir) + if gotErr == nil { + t.Errorf("ReadDir(%q): got no error, want error", dir) + } else if _, ok := gotErr.(*os.PathError); !ok { + t.Errorf("ReadDir(%q): got error with string %q and type %T, want os.PathError", dir, gotErr.Error(), gotErr) + } + } +} + +func TestOverlayPath(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "" + } +} +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- subdir5/deleted.txt -- +deleted +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +`) + + testCases := []struct { + path string + wantPath string + wantOK bool + }{ + {"subdir1/file1.txt", "subdir1/file1.txt", false}, + // OverlayPath returns false for directories + {"subdir2", "subdir2", false}, + {"subdir2/file2.txt", filepath.Join(cwd, "overlayfiles/subdir2_file2.txt"), true}, + // OverlayPath doesn't stat a file to see if it exists, so it happily returns + // the 'to' path and true even if the 'to' path doesn't exist on disk. + {"subdir3/doesntexist", filepath.Join(cwd, "this_file_doesnt_exist_anywhere"), true}, + // Like the subdir2/file2.txt case above, but subdir4 exists on disk, but subdir2 does not. + {"subdir4/this_file_is_overlaid.txt", filepath.Join(cwd, "overlayfiles/subdir4_this_file_is_overlaid.txt"), true}, + {"subdir5", "subdir5", false}, + {"subdir5/deleted.txt", "", true}, + } + + for _, tc := range testCases { + gotPath, gotOK := OverlayPath(tc.path) + if gotPath != tc.wantPath || gotOK != tc.wantOK { + t.Errorf("OverlayPath(%q): got %v, %v; want %v, %v", + tc.path, gotPath, gotOK, tc.wantPath, tc.wantOK) + } + } +} + +func TestOpen(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "", + "childoverlay/subdir1.txt/child.txt": "overlayfiles/child.txt", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "parentdeleted": "", + "parentdeleted/file.txt": "overlayfiles/parentdeleted_file.txt" + } +} +-- subdir11/deleted.txt -- +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- childoverlay/subdir1.txt -- +this file doesn't exist because the path +childoverlay/subdir1.txt/child.txt is in the overlay +-- subdir5/deleted.txt -- +deleted +-- parentdeleted -- +this will be deleted so that parentdeleted/file.txt can exist +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/child.txt -- +-- overlayfiles/subdir11 -- +11 +-- overlayfiles/parentdeleted_file.txt -- +this can exist because the parent directory is deleted +`) + + testCases := []struct { + path string + wantContents string + isErr bool + }{ + {"subdir1/file1.txt", "file 1\n", false}, + {"subdir2/file2.txt", "file 2\n", false}, + {"subdir3/doesntexist", "", true}, + {"subdir4/this_file_is_overlaid.txt", "99999999\n", false}, + {"subdir5/deleted.txt", "", true}, + {"parentoverwritten/subdir1/subdir2/subdir3/file.txt", "", true}, + {"childoverlay/subdir1.txt", "", true}, + {"subdir11", "11\n", false}, + {"parentdeleted/file.txt", "this can exist because the parent directory is deleted\n", false}, + } + + for _, tc := range testCases { + f, err := Open(tc.path) + if tc.isErr { + if err == nil { + f.Close() + t.Errorf("Open(%q): got no error, but want error", tc.path) + } + continue + } + if err != nil { + t.Errorf("Open(%q): got error %v, want nil", tc.path, err) + continue + } + contents, err := ioutil.ReadAll(f) + if err != nil { + t.Errorf("unexpected error reading contents of file: %v", err) + } + if string(contents) != tc.wantContents { + t.Errorf("contents of file opened with Open(%q): got %q, want %q", + tc.path, contents, tc.wantContents) + } + f.Close() + } +} + +func TestIsDirWithGoFiles(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "goinoverlay/file.go": "dummy", + "directory/removed/by/file": "dummy", + "directory_with_go_dir/dir.go/file.txt": "dummy", + "otherdirectory/deleted.go": "", + "nonexistentdirectory/deleted.go": "", + "textfile.txt/file.go": "dummy" + } +} +-- dummy -- +a destination file for the overlay entries to point to +contents don't matter for this test +-- nogo/file.txt -- +-- goondisk/file.go -- +-- goinoverlay/file.txt -- +-- directory/removed/by/file/in/overlay/file.go -- +-- otherdirectory/deleted.go -- +-- textfile.txt -- +`) + + testCases := []struct { + dir string + want bool + wantErr bool + }{ + {"nogo", false, false}, + {"goondisk", true, false}, + {"goinoverlay", true, false}, + {"directory/removed/by/file/in/overlay", false, false}, + {"directory_with_go_dir", false, false}, + {"otherdirectory", false, false}, + {"nonexistentdirectory", false, false}, + {"textfile.txt", true, false}, + } + + for _, tc := range testCases { + got, gotErr := IsDirWithGoFiles(tc.dir) + if tc.wantErr { + if gotErr == nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want non-nil error", tc.dir, got, gotErr) + } + continue + } + if gotErr != nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want nil error", tc.dir, got, gotErr) + } + if got != tc.want { + t.Errorf("IsDirWithGoFiles(%q) = %v; want %v", tc.dir, got, tc.want) + } + } +} diff --git a/src/cmd/go/internal/imports/scan.go b/src/cmd/go/internal/imports/scan.go index 3d9b6132b1..42ee49aaaa 100644 --- a/src/cmd/go/internal/imports/scan.go +++ b/src/cmd/go/internal/imports/scan.go @@ -6,16 +6,17 @@ package imports import ( "fmt" - "io/ioutil" "os" "path/filepath" "sort" "strconv" "strings" + + "cmd/go/internal/fsys" ) func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { - infos, err := ioutil.ReadDir(dir) + infos, err := fsys.ReadDir(dir) if err != nil { return nil, nil, err } @@ -49,7 +50,7 @@ func scanFiles(files []string, tags map[string]bool, explicitFiles bool) ([]stri numFiles := 0 Files: for _, name := range files { - r, err := os.Open(name) + r, err := fsys.Open(name) if err != nil { return nil, nil, err } diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index c36c8bd29b..3642de851a 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -17,6 +17,7 @@ import ( "time" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modfetch" "cmd/go/internal/par" "cmd/go/internal/search" @@ -438,57 +439,9 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. res := haveGoFilesCache.Do(dir, func() interface{} { - ok, err := isDirWithGoFiles(dir) + ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) return dir, res.haveGoFiles, res.err } - -func isDirWithGoFiles(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - defer f.Close() - - names, firstErr := f.Readdirnames(-1) - if firstErr != nil { - if fi, err := f.Stat(); err == nil && !fi.IsDir() { - return false, nil - } - - // Rewrite the error from ReadDirNames to include the path if not present. - // See https://golang.org/issue/38923. - var pe *os.PathError - if !errors.As(firstErr, &pe) { - firstErr = &os.PathError{Op: "readdir", Path: dir, Err: firstErr} - } - } - - for _, name := range names { - if strings.HasSuffix(name, ".go") { - info, err := os.Stat(filepath.Join(dir, name)) - if err == nil && info.Mode().IsRegular() { - // If any .go source file exists, the package exists regardless of - // errors for other source files. Leave further error reporting for - // later. - return true, nil - } - if firstErr == nil { - if os.IsNotExist(err) { - // If the file was concurrently deleted, or was a broken symlink, - // convert the error to an opaque error instead of one matching - // os.IsNotExist. - err = errors.New(err.Error()) - } - firstErr = err - } - } - } - - return false, firstErr -} diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 3344242489..e1b784860b 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -22,6 +22,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/lockedfile" "cmd/go/internal/modconv" "cmd/go/internal/modfetch" @@ -132,6 +133,10 @@ func Init() { return } + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding // the prompt in earlier versions is just too hard. diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index 4efef24152..868dbf5f9d 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -264,6 +264,7 @@ func (m *Match) MatchDirs() { } err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + // TODO(#39958): Handle walk for overlays. if err != nil { return err // Likely a permission error, which could interfere with matching. } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 86423f118c..21342ac8ba 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modfetch" "cmd/go/internal/modload" @@ -277,6 +278,8 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") + cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") + // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index d76574932e..1f15654c79 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -18,6 +18,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/objabi" @@ -145,7 +146,25 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle -trimprefix and other cases where + // tools depend on the names of the files that are passed in. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) } output, err = b.runOut(a, p.Dir, nil, args...) @@ -247,6 +266,8 @@ func (a *Action) trimpath() string { } } + // TODO(#39958): Add rewrite rules for overlaid files. + return rewrite } diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index b0d6133768..bab1935aca 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -9,6 +9,7 @@ package work import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modload" "cmd/internal/objabi" "cmd/internal/sys" @@ -24,6 +25,9 @@ func BuildInit() { modload.Init() instrumentInit() buildModeInit() + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } // Make sure -pkgdir is absolute, because we run commands // in different directories. -- cgit v1.2.1 From 1fb149fd640f2e83f17206aa6eb530d664b0b5ed Mon Sep 17 00:00:00 2001 From: witchard Date: Fri, 25 Sep 2020 14:09:42 +0000 Subject: cmd/go/internal/get: improve -insecure deprecation docs Updates #37519 Change-Id: I212607f1839b729d7da24b1258e56997b13ad830 GitHub-Last-Rev: db6d3c835bdf867a0b18f115276210e3a05902ed GitHub-Pull-Request: golang/go#41613 Reviewed-on: https://go-review.googlesource.com/c/go/+/257157 Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Trust: Jay Conrod Trust: Bryan C. Mills Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/get/get.go | 4 ++-- src/cmd/go/internal/modget/get.go | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ed2786879c..268962eca8 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -46,8 +46,8 @@ before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving custom domains using insecure schemes such as HTTP. Use with caution. This flag is deprecated and will be removed in a future version of go. -The GOINSECURE environment variable is usually a better alternative, since -it provides control over which modules may be retrieved using an insecure +The GOINSECURE environment variable should be used instead, since it +provides control over which packages may be retrieved using an insecure scheme. See 'go help environment' for details. The -t flag instructs get to also download the packages required to build diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index f1cf8b17a8..ea0e99af7d 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -115,13 +115,12 @@ require downgrading other dependencies, and 'go get' does this automatically as well. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. +custom domains using insecure schemes such as HTTP, and also bypassess +module sum validation using the checksum database. Use with caution. This flag is deprecated and will be removed in a future version of go. -The GOINSECURE environment variable is usually a better alternative, since -it provides control over which modules may be retrieved using an insecure -scheme. It should be noted that the -insecure flag also turns the module -checksum validation off. GOINSECURE does not do that, use GONOSUMDB. -See 'go help environment' for details. +To permit the use of insecure schemes, use the GOINSECURE environment +variable instead. To bypass module sum validation, use GOPRIVATE or +GONOSUMDB. See 'go help environment' for details. The second step is to download (if needed), build, and install the named packages. -- cgit v1.2.1 From db428ad7b61ed757671162054252b4326045e96c Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 17 Sep 2020 15:02:26 -0400 Subject: all: enable more tests on macOS/ARM64 Updates #38485. Change-Id: Iac96f5ffe88521fcb11eab306d0df6463bdce046 Reviewed-on: https://go-review.googlesource.com/c/go/+/256920 Trust: Cherry Zhang Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/work/build_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go index afed0fba72..904aee0684 100644 --- a/src/cmd/go/internal/work/build_test.go +++ b/src/cmd/go/internal/work/build_test.go @@ -221,10 +221,8 @@ func pkgImportPath(pkgpath string) *load.Package { // See https://golang.org/issue/18878. func TestRespectSetgidDir(t *testing.T) { switch runtime.GOOS { - case "darwin", "ios": - if runtime.GOARCH == "arm64" { - t.Skip("can't set SetGID bit with chmod on iOS") - } + case "ios": + t.Skip("can't set SetGID bit with chmod on iOS") case "windows", "plan9": t.Skip("chown/chmod setgid are not supported on Windows or Plan 9") } -- cgit v1.2.1 From 04b8a9fea57e37589d82410281f22ebde0027808 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 6 Oct 2020 14:42:15 -0700 Subject: all: implement GO386=softfloat Backstop support for non-sse2 chips now that 387 is gone. RELNOTE=yes Change-Id: Ib10e69c4a3654c15a03568f93393437e1939e013 Reviewed-on: https://go-review.googlesource.com/c/go/+/260017 Trust: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/cfg/cfg.go | 3 +++ src/cmd/go/internal/help/helpdoc.go | 3 +++ 2 files changed, 6 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 9169c12d8f..67d581f6e6 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -256,6 +256,7 @@ var ( // Used in envcmd.MkEnv and build ID computations. GOARM = envOr("GOARM", fmt.Sprint(objabi.GOARM)) + GO386 = envOr("GO386", objabi.GO386) GOMIPS = envOr("GOMIPS", objabi.GOMIPS) GOMIPS64 = envOr("GOMIPS64", objabi.GOMIPS64) GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64)) @@ -279,6 +280,8 @@ func GetArchEnv() (key, val string) { switch Goarch { case "arm": return "GOARM", GOARM + case "386": + return "GO386", GO386 case "mips", "mipsle": return "GOMIPS", GOMIPS case "mips64", "mips64le": diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index befa10a0e4..8dfabbaa4a 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -581,6 +581,9 @@ Architecture-specific environment variables: GOARM For GOARCH=arm, the ARM architecture for which to compile. Valid values are 5, 6, 7. + GO386 + For GOARCH=386, how to implement floating point instructions. + Valid values are sse2 (default), softfloat. GOMIPS For GOARCH=mips{,le}, whether to use floating point instructions. Valid values are hardfloat (default), softfloat. -- cgit v1.2.1 From 0941dc446e6b3028c77158728432086b5c06acf6 Mon Sep 17 00:00:00 2001 From: Eugene Kalinin Date: Tue, 25 Aug 2020 01:49:39 +0300 Subject: cmd/go: env -w validates GOTMPDIR value This change makes go env -w check if GOTMPDIR is an absolute path. If GOTMPDIR is not an absolute and not existing path there will be an error at every `work.Builder.Init()`. If `go env` has `-u/-w` as argument `work.Builder.Init()` is not called. `go env -w GOTMPDIR=` work in the same way as `go env -u GOTMPDIR`. Fixes #40932 Change-Id: I6b0662302eeace7f20460b6d26c6e59af1111da2 Reviewed-on: https://go-review.googlesource.com/c/go/+/250198 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Jay Conrod Trust: Bryan C. Mills Trust: Jay Conrod --- src/cmd/go/internal/envcmd/env.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index e1f2400f60..59d0ded658 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -203,10 +203,19 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { } // Do we need to call ExtraEnvVarsCostly, which is a bit expensive? - // Only if we're listing all environment variables ("go env") - // or the variables being requested are in the extra list. - needCostly := true - if len(args) > 0 { + needCostly := false + if *envU || *envW { + // We're overwriting or removing default settings, + // so it doesn't really matter what the existing settings are. + // + // Moreover, we haven't validated the new settings yet, so it is + // important that we NOT perform any actions based on them, + // such as initializing the builder to compute other variables. + } else if len(args) == 0 { + // We're listing all environment variables ("go env"), + // including the expensive ones. + needCostly = true + } else { needCostly = false for _, arg := range args { switch argKey(arg) { @@ -269,6 +278,13 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { } } + gotmp, okGOTMP := add["GOTMPDIR"] + if okGOTMP { + if !filepath.IsAbs(gotmp) && gotmp != "" { + base.Fatalf("go env -w: GOTMPDIR must be an absolute path") + } + } + updateEnvFile(add, nil) return } -- cgit v1.2.1 From 712cba3bf258c0c75bf1a638cb2119dbb11ed6c7 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Fri, 9 Oct 2020 15:16:13 -0400 Subject: cmd/go: ignore retracted versions when converting revisions to versions When a module author retracts a version, the go command should act as if it doesn't exist unless it's specifically requested. When converting a revision to a version, we should ignore tags for retracted versions. For example, if the tag v1.0.0 is retracted, and branch B points to the same revision, we should convert B to a pseudo-version, not v1.0.0. Similarly, if B points to a commit after v1.0.0, we should not use v1.0.0 as the base; we can use an earlier non-retracted tag or no base. Fixes #41700 Change-Id: Ia596b05b0780e5acfe6616a04e94d24bd342fbae Reviewed-on: https://go-review.googlesource.com/c/go/+/261079 Run-TryBot: Jay Conrod Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot Trust: Jay Conrod --- src/cmd/go/internal/modfetch/codehost/codehost.go | 5 +- src/cmd/go/internal/modfetch/codehost/git.go | 7 +- src/cmd/go/internal/modfetch/codehost/vcs.go | 2 +- src/cmd/go/internal/modfetch/coderepo.go | 78 +++++++++++++++++++++-- 4 files changed, 81 insertions(+), 11 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go index d85eddf767..df4cfdab1a 100644 --- a/src/cmd/go/internal/modfetch/codehost/codehost.go +++ b/src/cmd/go/internal/modfetch/codehost/codehost.go @@ -79,9 +79,8 @@ type Repo interface { ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) // RecentTag returns the most recent tag on rev or one of its predecessors - // with the given prefix and major version. - // An empty major string matches any major version. - RecentTag(rev, prefix, major string) (tag string, err error) + // with the given prefix. allowed may be used to filter out unwanted versions. + RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) // DescendsFrom reports whether rev or any of its ancestors has the given tag. // diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go index 31921324a7..5a35829c98 100644 --- a/src/cmd/go/internal/modfetch/codehost/git.go +++ b/src/cmd/go/internal/modfetch/codehost/git.go @@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F return missing, nil } -func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { info, err := r.Stat(rev) if err != nil { return "", err @@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { // NOTE: Do not replace the call to semver.Compare with semver.Max. // We want to return the actual tag, not a canonicalized version of it, // and semver.Max currently canonicalizes (see golang.org/issue/32700). - if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) { + continue + } + if semver.Compare(semtag, highest) > 0 { highest = semtag } } diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go index 7284557f4b..6278cb21e1 100644 --- a/src/cmd/go/internal/modfetch/codehost/vcs.go +++ b/src/cmd/go/internal/modfetch/codehost/vcs.go @@ -395,7 +395,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s return nil, vcsErrorf("ReadFileRevs not implemented") } -func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in // lock-inversion bugs. diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go index d043903336..d99a31d360 100644 --- a/src/cmd/go/internal/modfetch/coderepo.go +++ b/src/cmd/go/internal/modfetch/coderepo.go @@ -419,9 +419,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e tagPrefix = r.codeDir + "/" } + isRetracted, err := r.retractedVersions() + if err != nil { + isRetracted = func(string) bool { return false } + } + // tagToVersion returns the version obtained by trimming tagPrefix from tag. - // If the tag is invalid or a pseudo-version, tagToVersion returns an empty - // version. + // If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns + // an empty version. tagToVersion := func(tag string) (v string, tagIsCanonical bool) { if !strings.HasPrefix(tag, tagPrefix) { return "", false @@ -436,6 +441,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e if v == "" || !strings.HasPrefix(trimmed, v) { return "", false // Invalid or incomplete version (just vX or vX.Y). } + if isRetracted(v) { + return "", false + } if v == trimmed { tagIsCanonical = true } @@ -500,15 +508,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return checkGoMod() } + // Find the highest tagged version in the revision's history, subject to + // major version and +incompatible constraints. Use that version as the + // pseudo-version base so that the pseudo-version sorts higher. Ignore + // retracted versions. + allowedMajor := func(major string) func(v string) bool { + return func(v string) bool { + return (major == "" || semver.Major(v) == major) && !isRetracted(v) + } + } if pseudoBase == "" { var tag string if r.pseudoMajor != "" || canUseIncompatible() { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor) + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor)) } else { // Allow either v1 or v0, but not incompatible higher versions. - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1")) if tag == "" { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0")) } } pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid @@ -869,6 +886,57 @@ func (r *codeRepo) modPrefix(rev string) string { return r.modPath + "@" + rev } +func (r *codeRepo) retractedVersions() (func(string) bool, error) { + versions, err := r.Versions("") + if err != nil { + return nil, err + } + + for i, v := range versions { + if strings.HasSuffix(v, "+incompatible") { + versions = versions[:i] + break + } + } + if len(versions) == 0 { + return func(string) bool { return false }, nil + } + + var highest string + for i := len(versions) - 1; i >= 0; i-- { + v := versions[i] + if semver.Prerelease(v) == "" { + highest = v + break + } + } + if highest == "" { + highest = versions[len(versions)-1] + } + + data, err := r.GoMod(highest) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, err + } + retractions := make([]modfile.VersionInterval, len(f.Retract)) + for _, r := range f.Retract { + retractions = append(retractions, r.VersionInterval) + } + + return func(v string) bool { + for _, r := range retractions { + if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 { + return true + } + } + return false + }, nil +} + func (r *codeRepo) Zip(dst io.Writer, version string) error { if version != module.CanonicalVersion(version) { return fmt.Errorf("version %s is not canonical", version) -- cgit v1.2.1 From aeae46a7e51921b6af1dc1400bdc341fc1e568b0 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 22 Jun 2020 14:45:35 -0400 Subject: cmd/go: disable automatic go vet -unreachable during go test of std go test runs a limited number of vet checks by default. In the standard library, we run more, both to get additional checking for the standard library and to get experience with whether to enable any others by default. One that experience has shown us should not be enabled by default is go vet -unreachable. When you are testing, it is common to want to put an early return or a panic into code to bypass a section of code. That often causes unreachable code. It's incredibly frustrating if the result is an "unreachable code" error that keeps your test from completing. Change-Id: Ib194e87759eb65f5a193d771a9880b38d2fd3ba9 Reviewed-on: https://go-review.googlesource.com/c/go/+/240550 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/work/exec.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index e68b322c7d..17074afaf6 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -1052,17 +1052,28 @@ func (b *Builder) vet(ctx context.Context, a *Action) error { // This is OK as long as the packages that are farther down the // dependency tree turn on *more* analysis, as here. // (The unsafeptr check does not write any facts for use by - // later vet runs.) + // later vet runs, nor does unreachable.) if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Turn off -unsafeptr checks. + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. // Note that $GOROOT/src/buildall.bash // does the same for the misc-compile trybots // and should be updated if these flags are // changed here. - // - // There's too much unsafe.Pointer code - // that vet doesn't like in low-level packages - // like runtime, sync, and reflect. vetFlags = []string{"-unsafeptr=false"} + + // Also turn off -unreachable checks during go test. + // During testing it is very common to make changes + // like hard-coded forced returns or panics that make + // code unreachable. It's unreasonable to insist on files + // not having any unreachable code during "go test". + // (buildall.bash still runs with -unreachable enabled + // for the overall whole-tree scan.) + if cfg.CmdName == "test" { + vetFlags = append(vetFlags, "-unreachable=false") + } } // Note: We could decide that vet should compute export data for -- cgit v1.2.1 From 8b289a15e45328c9953a7c5c4ce1409c4297dee1 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 29 Jun 2020 10:38:06 -0400 Subject: cmd/go: remove Package.constraintIgnoredGoFiles Now all of IgnoredGoFiles is constraint-ignored Go files. Change-Id: I03001796c290708ab835526250c619dd667a8607 Reviewed-on: https://go-review.googlesource.com/c/go/+/240552 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/load/pkg.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 5cc77915e7..ffa083f2f0 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -185,7 +185,7 @@ type NoGoError struct { } func (e *NoGoError) Error() string { - if len(e.Package.constraintIgnoredGoFiles()) > 0 { + if len(e.Package.IgnoredGoFiles) > 0 { // Go files exist, but they were ignored due to build constraints. return "build constraints exclude all Go files in " + e.Package.Dir } @@ -2009,22 +2009,7 @@ func (p *Package) InternalXGoFiles() []string { // using absolute paths. "Possibly relevant" means that files are not excluded // due to build tags, but files with names beginning with . or _ are still excluded. func (p *Package) InternalAllGoFiles() []string { - return p.mkAbs(str.StringList(p.constraintIgnoredGoFiles(), p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) -} - -// constraintIgnoredGoFiles returns the list of Go files ignored for reasons -// other than having a name beginning with '.' or '_'. -func (p *Package) constraintIgnoredGoFiles() []string { - if len(p.IgnoredGoFiles) == 0 { - return nil - } - files := make([]string, 0, len(p.IgnoredGoFiles)) - for _, f := range p.IgnoredGoFiles { - if f != "" && f[0] != '.' && f[0] != '_' { - files = append(files, f) - } - } - return files + return p.mkAbs(str.StringList(p.IgnoredGoFiles, p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) } // usesSwig reports whether the package needs to run SWIG. -- cgit v1.2.1 From 7b77ff4c88befb1da9fb8b1cc7128a44ffb8a64f Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 29 Jun 2020 10:39:30 -0400 Subject: cmd/go: add IgnoredOtherFiles to go list; pass IgnoredFiles to vet Show constraint-ignored non-.go files in go list, as Package.IgnoredOtherFiles (same as go/build's IgnoredOtherFiles). Pass full list of ignored files to vet, to help buildtag checker. For #41184. Change-Id: I749868de9082cbbc1efbc59370783c8c82fe735f Reviewed-on: https://go-review.googlesource.com/c/go/+/240553 Trust: Russ Cox Reviewed-by: Jay Conrod --- src/cmd/go/internal/load/pkg.go | 29 ++++++++++++++++------------- src/cmd/go/internal/work/exec.go | 34 +++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 28 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index ffa083f2f0..5b3c2f0ff2 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -75,19 +75,20 @@ type PackagePublic struct { // Source files // If you add to this list you MUST add to p.AllFiles (below) too. // Otherwise file name security lists will not apply to any new additions. - GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string `json:",omitempty"` // .go source files that import "C" - CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles - IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints - CFiles []string `json:",omitempty"` // .c source files - CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files - MFiles []string `json:",omitempty"` // .m source files - HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files - FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files - SFiles []string `json:",omitempty"` // .s source files - SwigFiles []string `json:",omitempty"` // .swig files - SwigCXXFiles []string `json:",omitempty"` // .swigcxx files - SysoFiles []string `json:",omitempty"` // .syso system object files added to package + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go source files that import "C" + CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles + IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints + IgnoredOtherFiles []string `json:",omitempty"` // non-.go source files ignored due to build constraints + CFiles []string `json:",omitempty"` // .c source files + CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files + MFiles []string `json:",omitempty"` // .m source files + HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files + FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files + SFiles []string `json:",omitempty"` // .s source files + SwigFiles []string `json:",omitempty"` // .swig files + SwigCXXFiles []string `json:",omitempty"` // .swigcxx files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package // Cgo directives CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler @@ -127,6 +128,7 @@ func (p *Package) AllFiles() []string { p.CgoFiles, // no p.CompiledGoFiles, because they are from GoFiles or generated by us p.IgnoredGoFiles, + p.IgnoredOtherFiles, p.CFiles, p.CXXFiles, p.MFiles, @@ -330,6 +332,7 @@ func (p *Package) copyBuild(pp *build.Package) { p.GoFiles = pp.GoFiles p.CgoFiles = pp.CgoFiles p.IgnoredGoFiles = pp.IgnoredGoFiles + p.IgnoredOtherFiles = pp.IgnoredOtherFiles p.CFiles = pp.CFiles p.CXXFiles = pp.CXXFiles p.MFiles = pp.MFiles diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 17074afaf6..074bcc16c0 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -922,12 +922,13 @@ func (b *Builder) loadCachedSrcFiles(a *Action) error { // vetConfig is the configuration passed to vet describing a single package. type vetConfig struct { - ID string // package ID (example: "fmt [fmt.test]") - Compiler string // compiler name (gc, gccgo) - Dir string // directory containing package - ImportPath string // canonical import path ("package path") - GoFiles []string // absolute paths to package source files - NonGoFiles []string // absolute paths to package non-Go files + ID string // package ID (example: "fmt [fmt.test]") + Compiler string // compiler name (gc, gccgo) + Dir string // directory containing package + ImportPath string // canonical import path ("package path") + GoFiles []string // absolute paths to package source files + NonGoFiles []string // absolute paths to package non-Go files + IgnoredFiles []string // absolute paths to ignored source files ImportMap map[string]string // map import path in source code to package path PackageFile map[string]string // map package path to .a file with export data @@ -951,20 +952,23 @@ func buildVetConfig(a *Action, srcfiles []string) { } } + ignored := str.StringList(a.Package.IgnoredGoFiles, a.Package.IgnoredOtherFiles) + // Pass list of absolute paths to vet, // so that vet's error messages will use absolute paths, // so that we can reformat them relative to the directory // in which the go command is invoked. vcfg := &vetConfig{ - ID: a.Package.ImportPath, - Compiler: cfg.BuildToolchainName, - Dir: a.Package.Dir, - GoFiles: mkAbsFiles(a.Package.Dir, gofiles), - NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), - ImportPath: a.Package.ImportPath, - ImportMap: make(map[string]string), - PackageFile: make(map[string]string), - Standard: make(map[string]bool), + ID: a.Package.ImportPath, + Compiler: cfg.BuildToolchainName, + Dir: a.Package.Dir, + GoFiles: mkAbsFiles(a.Package.Dir, gofiles), + NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), + IgnoredFiles: mkAbsFiles(a.Package.Dir, ignored), + ImportPath: a.Package.ImportPath, + ImportMap: make(map[string]string), + PackageFile: make(map[string]string), + Standard: make(map[string]bool), } a.vetCfg = vcfg for i, raw := range a.Package.Internal.RawImports { -- cgit v1.2.1 From 15a11cedc6a9aac722369f134b76a157a559e050 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 6 Oct 2020 13:16:46 -0400 Subject: cmd/go: support walking through overlay directories Change-Id: I7d9d75aa1dbc34fec5073ca36091c626b9dd4920 Reviewed-on: https://go-review.googlesource.com/c/go/+/261537 Trust: Michael Matloob Trust: Jay Conrod Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/fsys/fsys.go | 78 +++++++- src/cmd/go/internal/fsys/fsys_test.go | 338 +++++++++++++++++++++++++++++++++- src/cmd/go/internal/modload/search.go | 3 +- src/cmd/go/internal/search/search.go | 8 +- 4 files changed, 419 insertions(+), 8 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index d64ce0aba1..489af93496 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -208,8 +208,8 @@ func IsDir(path string) (bool, error) { } // parentIsOverlayFile returns whether name or any of -// its parents are directories in the overlay, and the first parent found, -// including name itself, that's a directory in the overlay. +// its parents are files in the overlay, and the first parent found, +// including name itself, that's a file in the overlay. func parentIsOverlayFile(name string) (string, bool) { if overlay != nil { // Check if name can't possibly be a directory because @@ -385,6 +385,80 @@ func IsDirWithGoFiles(dir string) (bool, error) { return false, firstErr } +// walk recursively descends path, calling walkFn. Copied, with some +// modifications from path/filepath.walk. +func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + fis, readErr := ReadDir(path) + walkErr := walkFn(path, info, readErr) + // If readErr != nil, walk can't walk into this directory. + // walkErr != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of readErr and walkErr isn't nil, walk will return. + if readErr != nil || walkErr != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore readErr and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return walkErr + } + + for _, fi := range fis { + filename := filepath.Join(path, fi.Name()) + if walkErr = walk(filename, fi, walkFn); walkErr != nil { + if !fi.IsDir() || walkErr != filepath.SkipDir { + return walkErr + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + info, err := lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} + +// lstat implements a version of os.Lstat that operates on the overlay filesystem. +func lstat(path string) (os.FileInfo, error) { + cpath := canonicalize(path) + + if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + } + + node, ok := overlay[cpath] + if !ok { + // The file or directory is not overlaid. + return os.Lstat(cpath) + } + + switch { + case node.isDeleted(): + return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + case node.isDir(): + return fakeDir(filepath.Base(cpath)), nil + default: + fi, err := os.Lstat(node.actualFilePath) + if err != nil { + return nil, err + } + return fakeFile{name: filepath.Base(cpath), real: fi}, nil + } +} + // fakeFile provides an os.FileInfo implementation for an overlaid file, // so that the file has the name of the overlaid file, but takes all // other characteristics of the replacement file. diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 4b53059427..0c3069a6a2 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -3,10 +3,12 @@ package fsys import ( "cmd/go/internal/txtar" "encoding/json" + "errors" "fmt" "io/ioutil" "os" "path/filepath" + "reflect" "testing" ) @@ -22,7 +24,10 @@ func initOverlay(t *testing.T, config string) { if err != nil { t.Fatal(err) } - cwd = t.TempDir() + cwd = filepath.Join(t.TempDir(), "root") + if err := os.Mkdir(cwd, 0777); err != nil { + t.Fatal(err) + } if err := os.Chdir(cwd); err != nil { t.Fatal(err) } @@ -477,3 +482,334 @@ contents don't matter for this test } } } + +func TestWalk(t *testing.T) { + type file struct { + path string + name string + size int64 + mode os.FileMode + isDir bool + } + testCases := []struct { + name string + overlay string + root string + wantFiles []file + }{ + {"no overlay", ` +{} +-- file.txt -- +`, + ".", + []file{ + {".", "root", 0, os.ModeDir | 0700, true}, + {"file.txt", "file.txt", 0, 0600, false}, + }, + }, + {"overlay with different file", ` +{ + "Replace": { + "file.txt": "other.txt" + } +} +-- file.txt -- +-- other.txt -- +contents of other file +`, + ".", + []file{ + {".", "root", 0, os.ModeDir | 0500, true}, + {"file.txt", "file.txt", 23, 0600, false}, + {"other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new file", ` +{ + "Replace": { + "file.txt": "other.txt" + } +} +-- other.txt -- +contents of other file +`, + ".", + []file{ + {".", "root", 0, os.ModeDir | 0500, true}, + {"file.txt", "file.txt", 23, 0600, false}, + {"other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new directory", ` +{ + "Replace": { + "dir/file.txt": "other.txt" + } +} +-- other.txt -- +contents of other file +`, + ".", + []file{ + {".", "root", 0, os.ModeDir | 0500, true}, + {"dir", "dir", 0, os.ModeDir | 0500, true}, + {"dir" + string(filepath.Separator) + "file.txt", "file.txt", 23, 0600, false}, + {"other.txt", "other.txt", 23, 0600, false}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + + var got []file + Walk(tc.root, func(path string, info os.FileInfo, err error) error { + got = append(got, file{path, info.Name(), info.Size(), info.Mode(), info.IsDir()}) + return nil + }) + + if len(got) != len(tc.wantFiles) { + t.Errorf("Walk: saw %#v in walk; want %#v", got, tc.wantFiles) + } + for i := 0; i < len(got) && i < len(tc.wantFiles); i++ { + if got[i].path != tc.wantFiles[i].path { + t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, tc.wantFiles[i].path) + } + if got[i].name != tc.wantFiles[i].name { + t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name) + } + if got[i].mode&(os.ModeDir|0700) != tc.wantFiles[i].mode { + t.Errorf("mode&(os.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(os.ModeDir|0700), tc.wantFiles[i].mode) + } + if got[i].isDir != tc.wantFiles[i].isDir { + t.Errorf("isDir for file #%v in walk, got %v, want %v", i, got[i].isDir, tc.wantFiles[i].isDir) + } + if tc.wantFiles[i].isDir { + continue // don't check size for directories + } + if got[i].size != tc.wantFiles[i].size { + t.Errorf("size of file #%v in walk, got %v, want %v", i, got[i].size, tc.wantFiles[i].size) + } + } + }) + } +} + +func TestWalk_SkipDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "skipthisdir/file.go": "dummy.txt", + "dontskip/file.go": "dummy.txt", + "dontskip/skip/file.go": "dummy.txt" + } +} +-- dummy.txt -- +`) + + var seen []string + Walk(".", func(path string, info os.FileInfo, err error) error { + seen = append(seen, path) + if path == "skipthisdir" || path == filepath.Join("dontskip", "skip") { + return filepath.SkipDir + } + return nil + }) + + wantSeen := []string{".", "dontskip", filepath.Join("dontskip", "file.go"), filepath.Join("dontskip", "skip"), "dummy.txt", "skipthisdir"} + + if len(seen) != len(wantSeen) { + t.Errorf("paths seen in walk: got %v entries; want %v entries", len(seen), len(wantSeen)) + } + + for i := 0; i < len(seen) && i < len(wantSeen); i++ { + if seen[i] != wantSeen[i] { + t.Errorf("path #%v seen walking tree: want %q, got %q", i, seen[i], wantSeen[i]) + } + } +} + +func TestWalk_Error(t *testing.T) { + initOverlay(t, "{}") + + alreadyCalled := false + err := Walk("foo", func(path string, info os.FileInfo, err error) error { + if alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was called more than once") + } + alreadyCalled = true + return errors.New("returned from function") + }) + if !alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was never called") + + } + if err == nil { + t.Fatalf("Walk: got no error, want error") + } + if err.Error() != "returned from function" { + t.Fatalf("Walk: got error %v, want \"returned from function\" error", err) + } +} + +func TestWalk_Symlink(t *testing.T) { + initOverlay(t, `{ + "Replace": {"overlay_symlink": "symlink"} +} +-- dir/file --`) + + // Create symlink + if err := os.Symlink("dir", "symlink"); err != nil { + t.Error(err) + } + + testCases := []struct { + name string + dir string + wantFiles []string + }{ + {"control", "dir", []string{"dir", "dir" + string(filepath.Separator) + "file"}}, + // ensure Walk doesn't wolk into the directory pointed to by the symlink + // (because it's supposed to use Lstat instead of Stat. + {"symlink_to_dir", "symlink", []string{"symlink"}}, + {"overlay_to_symlink_to_dir", "overlay_symlink", []string{"overlay_symlink"}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var got []string + + err := Walk(tc.dir, func(path string, info os.FileInfo, err error) error { + got = append(got, path) + if err != nil { + t.Errorf("walkfn: got non nil err argument: %v, want nil err argument", err) + } + return nil + }) + if err != nil { + t.Errorf("Walk: got error %q, want nil", err) + } + + if !reflect.DeepEqual(got, tc.wantFiles) { + t.Errorf("files examined by walk: got %v, want %v", got, tc.wantFiles) + } + }) + } + +} + +func TestLstat(t *testing.T) { + type file struct { + name string + size int64 + mode os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | os.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | os.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := lstat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("lstat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("lstat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("lstat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(os.ModeDir|0700) != tc.want.mode { + t.Errorf("lstat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("lstat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("lstat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index a9bee0af4e..0f82026732 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -12,6 +12,7 @@ import ( "strings" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/search" @@ -53,7 +54,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil { m.AddError(err) return nil diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index 868dbf5f9d..b1d2a9376b 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -7,6 +7,7 @@ package search import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "fmt" "go/build" "os" @@ -127,7 +128,7 @@ func (m *Match) MatchPackages() { if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -263,8 +264,7 @@ func (m *Match) MatchDirs() { } } - err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - // TODO(#39958): Handle walk for overlays. + err := fsys.Walk(dir, func(path string, fi os.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -273,7 +273,7 @@ func (m *Match) MatchDirs() { } top := false if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, + // Walk starts at dir and recurses. For the recursive case, // the path is the result of filepath.Join, which calls filepath.Clean. // The initial case is not Cleaned, though, so we do this explicitly. // -- cgit v1.2.1 From 6c0135d3772e710328c751fbc704927931f129ca Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sun, 11 Oct 2020 16:27:25 -0400 Subject: cmd/go: don't always link in cgo for PIE Internal linking for PIE is now supported and enabled by default on some platforms, for which cgo is not needed. Don't always bring in cgo. Change-Id: I043ed436f0e6a3acbcc53ec543f06e193d614b36 Reviewed-on: https://go-review.googlesource.com/c/go/+/261498 Trust: Cherry Zhang Run-TryBot: Cherry Zhang TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/load/pkg.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 5b3c2f0ff2..db2434260f 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -33,6 +33,7 @@ import ( "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/sys" ) var IgnoreImports bool // control whether we ignore imports in packages @@ -1968,7 +1969,7 @@ func externalLinkingForced(p *Package) bool { // external linking mode, as of course does // -ldflags=-linkmode=external. External linking mode forces // an import of runtime/cgo. - pieCgo := cfg.BuildBuildmode == "pie" + pieCgo := cfg.BuildBuildmode == "pie" && !sys.InternalLinkPIESupported(cfg.BuildContext.GOOS, cfg.BuildContext.GOARCH) linkmodeExternal := false if p != nil { ldflags := BuildLdflags.For(p) -- cgit v1.2.1 From 58e51b1e620167cc22ca7143c395cb63db5640a8 Mon Sep 17 00:00:00 2001 From: Fazlul Shahriar Date: Tue, 13 Oct 2020 11:36:11 -0400 Subject: cmd/go/internal/fsys: skip symlink test on Plan 9 Fixes #41950 Fixes #41954 Change-Id: I95d97f076fa928f3638309b78748d7ccc7277b14 Reviewed-on: https://go-review.googlesource.com/c/go/+/261897 Trust: Jay Conrod Run-TryBot: Bryan C. Mills Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot --- src/cmd/go/internal/fsys/fsys_test.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 0c3069a6a2..6cf59fba47 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "internal/testenv" "io/ioutil" "os" "path/filepath" @@ -654,6 +655,8 @@ func TestWalk_Error(t *testing.T) { } func TestWalk_Symlink(t *testing.T) { + testenv.MustHaveSymlink(t) + initOverlay(t, `{ "Replace": {"overlay_symlink": "symlink"} } -- cgit v1.2.1 From 3a65abfbdac7ab29f693d69bd1eb12b2148a11ae Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 29 Sep 2020 17:45:02 -0400 Subject: cmd/go: adjust ImportMissingError when module lookup is disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, ImportMissingError said "cannot find module providing package …" even when we didn't even attempt to find such a module. Now, we write "no module requirement provides package …" when we did not attempt to identify a suitable module, and suggest either 'go mod tidy' or 'go get -d' as appropriate. Fixes #41576 Change-Id: I979bb999da4066828c54d99a310ea66bb31032ec Reviewed-on: https://go-review.googlesource.com/c/go/+/258298 Trust: Bryan C. Mills Trust: Jay Conrod Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Michael Matloob Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/import.go | 40 ++++++++++++++++++++--------------- src/cmd/go/internal/modload/load.go | 25 ++++++++++++++++------ 2 files changed, 41 insertions(+), 24 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 3642de851a..76fe6745d9 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -26,13 +26,15 @@ import ( "golang.org/x/mod/semver" ) -var errImportMissing = errors.New("import missing") - type ImportMissingError struct { Path string Module module.Version QueryErr error + // inAll indicates whether Path is in the "all" package pattern, + // and thus would be added by 'go mod tidy'. + inAll bool + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string @@ -46,7 +48,19 @@ func (e *ImportMissingError) Error() string { if e.QueryErr != nil { return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) } - return "cannot find module providing package " + e.Path + if cfg.BuildMod == "mod" { + return "cannot find module providing package " + e.Path + } + + suggestion := "" + if !HasModRoot() { + suggestion = ": working directory is not part of a module" + } else if e.inAll { + suggestion = "; try 'go mod tidy' to add it" + } else { + suggestion = fmt.Sprintf("; try 'go get -d %s' to add it", e.Path) + } + return fmt.Sprintf("no required module provides package %s%s", e.Path, suggestion) } if e.newMissingVersion != "" { @@ -132,7 +146,7 @@ func (e *invalidImportError) Unwrap() error { // like "C" and "unsafe". // // If the package cannot be found in the current build list, -// importFromBuildList returns errImportMissing as the error. +// importFromBuildList returns an *ImportMissingError. func importFromBuildList(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") @@ -144,6 +158,10 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di // There's no directory for import "C" or import "unsafe". return module.Version{}, "", nil } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", &invalidImportError{importPath: path, err: err} + } // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) @@ -212,7 +230,7 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - return module.Version{}, "", errImportMissing + return module.Version{}, "", &ImportMissingError{Path: path} } // queryImport attempts to locate a module that can be added to the current @@ -220,13 +238,6 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di func queryImport(ctx context.Context, path string) (module.Version, error) { pathIsStd := search.IsStandardImportPath(path) - if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, &ImportMissingError{ - Path: path, - QueryErr: errors.New("working directory is not part of a module"), - } - } - // Not on build list. // To avoid spurious remote fetches, next try the latest replacement for each // module (golang.org/issue/26241). This should give a useful message @@ -291,11 +302,6 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { } } - // Before any further lookup, check that the path is valid. - if err := module.CheckImportPath(path); err != nil { - return module.Version{}, &invalidImportError{importPath: path, err: err} - } - if pathIsStd { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 9194f9cc7c..4ddb817cf1 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -270,11 +270,19 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // Report errors, if any. checkMultiplePaths() for _, pkg := range loaded.pkgs { - if pkg.err != nil && !opts.SilenceErrors { - if opts.AllowErrors { - fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) - } else { - base.Errorf("%s: %v", pkg.stackText(), pkg.err) + if pkg.err != nil { + if pkg.flags.has(pkgInAll) { + if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) { + imErr.inAll = true + } + } + + if !opts.SilenceErrors { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } } } if !pkg.isTest() { @@ -809,7 +817,7 @@ func loadFromRoots(params loaderParams) *loader { ld.buildStacks() - if !ld.resolveMissing { + if !ld.resolveMissing || (!HasModRoot() && !allowMissingModuleImports) { // We've loaded as much as we can without resolving missing imports. break } @@ -872,12 +880,15 @@ func loadFromRoots(params loaderParams) *loader { func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { var needPkgs []*loadPkg for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue + } if pkg.isTest() { // If we are missing a test, we are also missing its non-test version, and // we should only add the missing import once. continue } - if pkg.err != errImportMissing { + if !errors.As(pkg.err, new(*ImportMissingError)) { // Leave other errors for Import or load.Packages to report. continue } -- cgit v1.2.1 From c9211577eb77df9c51f0565f1da7d20ff91d59df Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 2 Oct 2020 16:25:17 -0400 Subject: cmd/go/internal/modfetch: remove error return from Lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We generally don't care about errors in resolving a repo if the result we're looking for is already in the module cache. Moreover, we can avoid some expense in initializing the repo if all of the methods we plan to call on it hit in the cache — especially when using GOPROXY=direct. This also incidentally fixes a possible (but rare) bug in Download: we had forgotten to reset the downloaded file in case the Zip method returned an error after writing a nonzero number of bytes. For #37438 Change-Id: Ib64f10f763f6d1936536b8e1f7d31ed1b463e955 Reviewed-on: https://go-review.googlesource.com/c/go/+/259158 Trust: Bryan C. Mills Trust: Jay Conrod Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Michael Matloob Reviewed-by: Jay Conrod --- src/cmd/go/internal/modfetch/cache.go | 70 ++++++++++++--------------- src/cmd/go/internal/modfetch/coderepo_test.go | 28 +++-------- src/cmd/go/internal/modfetch/fetch.go | 22 +++++++-- src/cmd/go/internal/modfetch/repo.go | 38 ++++++++++----- src/cmd/go/internal/modload/mvs.go | 6 +-- src/cmd/go/internal/modload/query.go | 10 ++-- 6 files changed, 87 insertions(+), 87 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go index e3074b775e..6eadb026c9 100644 --- a/src/cmd/go/internal/modfetch/cache.go +++ b/src/cmd/go/internal/modfetch/cache.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -155,16 +156,30 @@ func SideLock() (unlock func(), err error) { type cachingRepo struct { path string cache par.Cache // cache for all operations - r Repo + + once sync.Once + initRepo func() (Repo, error) + r Repo } -func newCachingRepo(r Repo) *cachingRepo { +func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo { return &cachingRepo{ - r: r, - path: r.ModulePath(), + path: path, + initRepo: initRepo, } } +func (r *cachingRepo) repo() Repo { + r.once.Do(func() { + var err error + r.r, err = r.initRepo() + if err != nil { + r.r = errRepo{r.path, err} + } + }) + return r.r +} + func (r *cachingRepo) ModulePath() string { return r.path } @@ -175,7 +190,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { err error } c := r.cache.Do("versions:"+prefix, func() interface{} { - list, err := r.r.Versions(prefix) + list, err := r.repo().Versions(prefix) return cached{list, err} }).(cached) @@ -197,7 +212,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - info, err = r.r.Stat(rev) + info, err = r.repo().Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, // then save the information under the proper version, for future use. @@ -224,7 +239,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - info, err := r.r.Latest() + info, err := r.repo().Latest() // Save info for likely future Stat call. if err == nil { @@ -258,7 +273,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { return cached{text, nil} } - text, err = r.r.GoMod(version) + text, err = r.repo().GoMod(version) if err == nil { if err := checkGoMod(r.path, version, text); err != nil { return cached{text, err} @@ -277,26 +292,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { } func (r *cachingRepo) Zip(dst io.Writer, version string) error { - return r.r.Zip(dst, version) -} - -// Stat is like Lookup(path).Stat(rev) but avoids the -// repository path resolution in Lookup if the result is -// already cached on local disk. -func Stat(proxy, path, rev string) (*RevInfo, error) { - _, info, err := readDiskStat(path, rev) - if err == nil { - return info, nil - } - repo, err := Lookup(proxy, path) - if err != nil { - return nil, err - } - return repo.Stat(rev) + return r.repo().Zip(dst, version) } -// InfoFile is like Stat but returns the name of the file containing -// the cached information. +// InfoFile is like Lookup(path).Stat(version) but returns the name of the file +// containing the cached information. func InfoFile(path, version string) (string, error) { if !semver.IsValid(version) { return "", fmt.Errorf("invalid version %q", version) @@ -307,10 +307,7 @@ func InfoFile(path, version string) (string, error) { } err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - _, err = repo.Stat(version) - } + _, err := Lookup(proxy, path).Stat(version) return err }) if err != nil { @@ -336,11 +333,7 @@ func GoMod(path, rev string) ([]byte, error) { rev = info.Version } else { err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err != nil { - return err - } - info, err := repo.Stat(rev) + info, err := Lookup(proxy, path).Stat(rev) if err == nil { rev = info.Version } @@ -357,11 +350,8 @@ func GoMod(path, rev string) ([]byte, error) { return data, nil } - err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - data, err = repo.GoMod(rev) - } + err = TryProxies(func(proxy string) (err error) { + data, err = Lookup(proxy, path).GoMod(rev) return err }) return data, err diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go index f69c193b86..28c5e67a28 100644 --- a/src/cmd/go/internal/modfetch/coderepo_test.go +++ b/src/cmd/go/internal/modfetch/coderepo_test.go @@ -60,7 +60,6 @@ var altVgotests = map[string]string{ type codeRepoTest struct { vcs string path string - lookErr string mpath string rev string err string @@ -332,9 +331,9 @@ var codeRepoTests = []codeRepoTest{ // package in subdirectory - custom domain // In general we can't reject these definitively in Lookup, // but gopkg.in is special. - vcs: "git", - path: "gopkg.in/yaml.v2/abc", - lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"", + vcs: "git", + path: "gopkg.in/yaml.v2/abc", + err: "invalid module path \"gopkg.in/yaml.v2/abc\"", }, { // package in subdirectory - github @@ -440,16 +439,7 @@ func TestCodeRepo(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if tt.lookErr != "" { - if err != nil && err.Error() == tt.lookErr { - return - } - t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr) - } - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) if tt.mpath == "" { tt.mpath = tt.path @@ -685,10 +675,7 @@ func TestCodeRepoVersions(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) list, err := repo.Versions(tt.prefix) if err != nil { t.Fatalf("Versions(%q): %v", tt.prefix, err) @@ -763,10 +750,7 @@ func TestLatest(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) info, err := repo.Latest() if err != nil { if tt.err != "" { diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 1d90002faa..599419977a 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -233,12 +233,28 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e } }() + var unrecoverableErr error err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, mod.Path) + if unrecoverableErr != nil { + return unrecoverableErr + } + repo := Lookup(proxy, mod.Path) + err := repo.Zip(f, mod.Version) if err != nil { - return err + // Zip may have partially written to f before failing. + // (Perhaps the server crashed while sending the file?) + // Since we allow fallback on error in some cases, we need to fix up the + // file to be empty again for the next attempt. + if _, err := f.Seek(0, io.SeekStart); err != nil { + unrecoverableErr = err + return err + } + if err := f.Truncate(0); err != nil { + unrecoverableErr = err + return err + } } - return repo.Zip(f, mod.Version) + return err }) if err != nil { return err diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index eed4dd4258..4936ec11aa 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -188,27 +188,26 @@ type lookupCacheKey struct { // // A successful return does not guarantee that the module // has any defined versions. -func Lookup(proxy, path string) (Repo, error) { +func Lookup(proxy, path string) Repo { if traceRepo { defer logCall("Lookup(%q, %q)", proxy, path)() } type cached struct { - r Repo - err error + r Repo } c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { - r, err := lookup(proxy, path) - if err == nil { - if traceRepo { + r := newCachingRepo(path, func() (Repo, error) { + r, err := lookup(proxy, path) + if err == nil && traceRepo { r = newLoggingRepo(r) } - r = newCachingRepo(r) - } - return cached{r, err} + return r, err + }) + return cached{r} }).(cached) - return c.r, c.err + return c.r } // lookup returns the module with the given module path. @@ -228,7 +227,7 @@ func lookup(proxy, path string) (r Repo, err error) { switch proxy { case "off": - return nil, errProxyOff + return errRepo{path, errProxyOff}, nil case "direct": return lookupDirect(path) case "noproxy": @@ -407,6 +406,23 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error { return l.r.Zip(dst, version) } +// errRepo is a Repo that returns the same error for all operations. +// +// It is useful in conjunction with caching, since cache hits will not attempt +// the prohibited operations. +type errRepo struct { + modulePath string + err error +} + +func (r errRepo) ModulePath() string { return r.modulePath } + +func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err } +func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err } +func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err } +func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } +func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } + // A notExistError is like os.ErrNotExist, but with a custom message type notExistError struct { err error diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 24856260d4..65329524f9 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -77,11 +77,7 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return err - } - allVersions, err := repo.Versions("") + allVersions, err := modfetch.Lookup(proxy, path).Versions("") if err != nil { return err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e75d901ec6..44076fb615 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -229,14 +229,15 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. - info, err := modfetch.Stat(proxy, path, query) + repo := modfetch.Lookup(proxy, path) + info, err := repo.Stat(query) if err != nil { queryErr := err // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. if canonicalQuery != "" && query != canonicalQuery { - info, err = modfetch.Stat(proxy, path, canonicalQuery) + info, err = repo.Stat(canonicalQuery) if err != nil && !errors.Is(err, os.ErrNotExist) { return info, err } @@ -266,10 +267,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } // Load versions and execute query. - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return nil, err - } + repo := modfetch.Lookup(proxy, path) versions, err := repo.Versions(prefix) if err != nil { return nil, err -- cgit v1.2.1 From 52669c4a689a15c4f307e11609d6237ac506a3df Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 12 Oct 2020 23:35:25 -0400 Subject: cmd/go: update go list docs for IgnoredOtherFiles Change-Id: I8eb7f34754c7be899d389fe807af65aa5fd5bbc4 Reviewed-on: https://go-review.googlesource.com/c/go/+/261957 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/list/list.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 33409eb774..732cebc8cb 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -71,21 +71,22 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is: DepOnly bool // package is only a dependency, not explicitly listed // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go source files that import "C" - CompiledGoFiles []string // .go files presented to compiler (when using -compiled) - IgnoredGoFiles []string // .go source files ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - FFiles []string // .f, .F, .for and .f90 Fortran source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - TestGoFiles []string // _test.go files in package - XTestGoFiles []string // _test.go files outside package + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + CompiledGoFiles []string // .go files presented to compiler (when using -compiled) + IgnoredGoFiles []string // .go source files ignored due to build constraints + IgnoredOtherFiles []string // non-.go source files ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package // Cgo directives CgoCFLAGS []string // cgo: flags for C compiler -- cgit v1.2.1 From e4ec30965b9ca629922e83b8d335224ae4bdf062 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 12 Oct 2020 13:44:21 -0400 Subject: cmd/link: support internal linking on darwin/arm64 Add support of internal linking on darwin/arm64 (macOS). Still incomplete. Pure Go binaries work. Cgo doesn't. TLS is not set up when cgo is not used (as before) (so asynchronous preemption is not enabled). Internal linking is not enabled by default but can be requested via -ldflags=-linkmode=internal. Updates #38485. Change-Id: I1e0c81b6028edcb1ac26dcdafeb9bb3f788cf732 Reviewed-on: https://go-review.googlesource.com/c/go/+/261643 Trust: Cherry Zhang Reviewed-by: Than McIntosh --- src/cmd/go/internal/load/pkg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index db2434260f..f73b79d089 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1948,6 +1948,10 @@ func LinkerDeps(p *Package) []string { // externalLinkingForced reports whether external linking is being // forced even for programs that do not use cgo. func externalLinkingForced(p *Package) bool { + if !cfg.BuildContext.CgoEnabled { + return false + } + // Some targets must use external linking even inside GOROOT. switch cfg.BuildContext.GOOS { case "android": @@ -1960,9 +1964,6 @@ func externalLinkingForced(p *Package) bool { } } - if !cfg.BuildContext.CgoEnabled { - return false - } // Currently build modes c-shared, pie (on systems that do not // support PIE with internal linking mode (currently all // systems: issue #18968)), plugin, and -linkshared force -- cgit v1.2.1 From aa161e799df7e1eba99d2be10271e76b6f758142 Mon Sep 17 00:00:00 2001 From: Obeyda Djeffal Date: Thu, 16 Apr 2020 12:45:37 +0100 Subject: cmd/go: make sure CC and CXX are absolute Add check in cmd/go/internal/work.BuildInit and cmd/go/internal/envcmd.checkEnvWrite. Fixes #38372 Change-Id: I196ea93a0469e4667ef785f7c1dc4574bdf7ff78 Reviewed-on: https://go-review.googlesource.com/c/go/+/228517 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Jay Conrod Trust: Jay Conrod Trust: Michael Matloob --- src/cmd/go/internal/envcmd/env.go | 5 +++++ src/cmd/go/internal/work/init.go | 7 +++++++ 2 files changed, 12 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 59d0ded658..b5a48558fa 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -424,6 +424,11 @@ func checkEnvWrite(key, val string) error { if !filepath.IsAbs(val) && val != "" { return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val) } + // Make sure CC and CXX are absolute paths + case "CC", "CXX": + if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) { + return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val) + } } if !utf8.ValidString(val) { diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index bab1935aca..81c4fb7465 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -41,6 +41,13 @@ func BuildInit() { cfg.BuildPkgdir = p } + // Make sure CC and CXX are absolute paths + for _, key := range []string{"CC", "CXX"} { + if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { + base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) + } + } + // For each experiment that has been enabled in the toolchain, define a // build tag with the same name but prefixed by "goexperiment." which can be // used for compiling alternative files for the experiment. This allows -- cgit v1.2.1 From 21e441c461a79be41de20a99e6001098142946e6 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 14 Oct 2020 16:21:39 -0400 Subject: cmd/go: rewrite paths for overlaid files using -trimpath Pass the trimpath flag to cmd/compile to use the correct file paths for files that are overlaid: that is, the "destination" path in the overlay's Replace mapping rather than the "source" path. Also fix paths to go source files provided to the gccgo compiler. For #39958 Change-Id: I3741aeb2272bd0d5aa32cb28133b61e58264fd39 Reviewed-on: https://go-review.googlesource.com/c/go/+/257198 Trust: Michael Matloob Trust: Bryan C. Mills Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/work/exec.go | 2 ++ src/cmd/go/internal/work/gc.go | 25 +++++++++++++++++-------- src/cmd/go/internal/work/gccgo.go | 33 +++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 12 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 074bcc16c0..824a4b5a0a 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2214,6 +2214,8 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s // when -trimpath is enabled. if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { if cfg.BuildTrimpath { + // TODO(#39958): handle overlays + // Keep in sync with Action.trimpath. // The trimmed paths are a little different, but we need to trim in the // same situations. diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 1f15654c79..56ad1872be 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -152,8 +152,6 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s // so these paths can be handed directly to tools. // Deleted files won't show up in when scanning directories earlier, // so OverlayPath will never return "" (meaning a deleted file) here. - // TODO(#39958): Handle -trimprefix and other cases where - // tools depend on the names of the files that are passed in. // TODO(#39958): Handle cases where the package directory // doesn't exist on disk (this can happen when all the package's // files are in an overlay): the code expects the package directory @@ -167,7 +165,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s args = append(args, f) } - output, err = b.runOut(a, p.Dir, nil, args...) + output, err = b.runOut(a, base.Cwd, nil, args...) return ofile, output, err } @@ -256,17 +254,28 @@ func (a *Action) trimpath() string { } rewrite := objdir + "=>" - // For "go build -trimpath", rewrite package source directory - // to a file system-independent path (just the import path). + rewriteDir := a.Package.Dir if cfg.BuildTrimpath { if m := a.Package.Module; m != nil && m.Version != "" { - rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) } else { - rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath + rewriteDir = a.Package.ImportPath } + rewrite += ";" + a.Package.Dir + "=>" + rewriteDir } - // TODO(#39958): Add rewrite rules for overlaid files. + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + overlayPath, ok := fsys.OverlayPath(filepath.Join(a.Package.Dir, filename)) + if !ok { + continue + } + rewrite += ";" + overlayPath + "=>" + filepath.Join(rewriteDir, filename) + } + } return rewrite } diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go index dd5adf2d7b..ade8964b7c 100644 --- a/src/cmd/go/internal/work/gccgo.go +++ b/src/cmd/go/internal/work/gccgo.go @@ -15,6 +15,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/pkgpath" @@ -93,13 +94,37 @@ func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg args = append(args, "-I", root) } } - if cfg.BuildTrimpath && b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { - args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") - args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + + if b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + if cfg.BuildTrimpath { + args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if fsys.OverlayFile != "" { + for _, name := range gofiles { + absPath := mkAbs(p.Dir, name) + overlayPath, ok := fsys.OverlayPath(absPath) + if !ok { + continue + } + toPath := absPath + // gccgo only applies the last matching rule, so also handle the case where + // BuildTrimpath is true and the path is relative to base.Cwd. + if cfg.BuildTrimpath && str.HasFilePathPrefix(toPath, base.Cwd) { + toPath = "." + toPath[len(base.Cwd):] + } + args = append(args, "-ffile-prefix-map="+overlayPath+"="+toPath) + } + } } + args = append(args, a.Package.Internal.Gccgoflags...) for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) + f := mkAbs(p.Dir, f) + // Overlay files if necessary. + // See comment on gctoolchain.gc about overlay TODOs + f, _ = fsys.OverlayPath(f) + args = append(args, f) } output, err = b.runOut(a, p.Dir, nil, args) -- cgit v1.2.1 From 8ee4d6e1bf1e76a4eac627d5fc9b0d0332d209e2 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 15 Oct 2020 15:53:09 -0400 Subject: cmd/go/internal/modload: move fetch to import.go From a comment in CL 262341. It makes more sense in import.go than in mvs.go. Change-Id: If4dfa1091077e110c5041bc849d99bc0be2bd8e2 Reviewed-on: https://go-review.googlesource.com/c/go/+/262780 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modload/import.go | 39 ++++++++++++++++++++++++++++++++ src/cmd/go/internal/modload/mvs.go | 42 ----------------------------------- 2 files changed, 39 insertions(+), 42 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 76fe6745d9..8641cfec08 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -451,3 +451,42 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile return dir, res.haveGoFiles, res.err } + +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil + } + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) + } + return dir, true, module.VersionError(mod, err) + } + return dir, true, nil + } + mod = r + } + + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 65329524f9..76a1d8a12a 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -7,9 +7,6 @@ package modload import ( "context" "errors" - "fmt" - "os" - "path/filepath" "sort" "cmd/go/internal/modfetch" @@ -125,42 +122,3 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { } return module.Version{Path: m.Path, Version: "none"}, nil } - -// fetch downloads the given module (or its replacement) -// and returns its location. -// -// The isLocal return value reports whether the replacement, -// if any, is local to the filesystem. -func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil - } - if r := Replacement(mod); r.Path != "" { - if r.Version == "" { - dir = r.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - // Ensure that the replacement directory actually exists: - // dirInModule does not report errors for missing modules, - // so if we don't report the error now, later failures will be - // very mysterious. - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - // Semantically the module version itself “exists” — we just don't - // have its source code. Remove the equivalence to os.ErrNotExist, - // and make the message more concise while we're at it. - err = fmt.Errorf("replacement directory %s does not exist", r.Path) - } else { - err = fmt.Errorf("replacement directory %s: %w", r.Path, err) - } - return dir, true, module.VersionError(mod, err) - } - return dir, true, nil - } - mod = r - } - - dir, err = modfetch.Download(ctx, mod) - return dir, false, err -} -- cgit v1.2.1 From ff052737a946b5f9381dc054d61857ee4d500899 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 28 Sep 2020 20:59:47 -0400 Subject: cmd/go/internal/modload: allow 'go get' to use replaced versions 'go mod tidy' has been able to use replaced versions since CL 152739, but 'go get' failed for many of the same paths. Now that we are recommending 'go get' more aggressively due to #40728, we should make that work too. In the future, we might consider factoring out the new replacementRepo type so that 'go list' can report the new versions as well. For #41577 For #41416 For #37438 Updates #26241 Change-Id: I9140c556424b584fdd9bdd0a747842774664a7d8 Reviewed-on: https://go-review.googlesource.com/c/go/+/258220 Trust: Bryan C. Mills Trust: Jay Conrod Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Michael Matloob Reviewed-by: Jay Conrod --- src/cmd/go/internal/modfetch/repo.go | 9 + src/cmd/go/internal/modget/get.go | 17 + src/cmd/go/internal/modget/mvs.go | 2 +- src/cmd/go/internal/modload/import.go | 93 ++--- src/cmd/go/internal/modload/modfile.go | 54 ++- src/cmd/go/internal/modload/query.go | 546 +++++++++++++++++++++--------- src/cmd/go/internal/modload/query_test.go | 22 +- 7 files changed, 521 insertions(+), 222 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 4936ec11aa..c7cf5595bd 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -32,8 +32,17 @@ type Repo interface { // Versions lists all known versions with the given prefix. // Pseudo-versions are not included. + // // Versions should be returned sorted in semver order // (implementations can use SortVersions). + // + // Versions returns a non-nil error only if there was a problem + // fetching the list of versions: it may return an empty list + // along with a nil error if the list of matching versions + // is known to be empty. + // + // If the underlying repository does not exist, + // Versions returns an error matching errors.Is(_, os.NotExist). Versions(prefix string) ([]string, error) // Stat returns information about the revision rev. diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index ea0e99af7d..9e2fb8e408 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -874,6 +874,8 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc allowed := modload.CheckAllowed if modload.IsRevisionQuery(vers) { allowed = modload.CheckExclusions + } else if vers == "upgrade" || vers == "patch" { + allowed = checkAllowedOrCurrent(prevM.Version) } // If the query must be a module path, try only that module path. @@ -981,3 +983,18 @@ func logOncef(format string, args ...interface{}) { fmt.Fprintln(os.Stderr, msg) } } + +// checkAllowedOrCurrent is like modload.CheckAllowed, but always allows the +// current version (even if it is retracted or otherwise excluded). +func checkAllowedOrCurrent(current string) modload.AllowedFunc { + if current == "" { + return modload.CheckAllowed + } + + return func(ctx context.Context, m module.Version) error { + if m.Version == current { + return nil + } + return modload.CheckAllowed(ctx, m) + } +} diff --git a/src/cmd/go/internal/modget/mvs.go b/src/cmd/go/internal/modget/mvs.go index 19fffd2947..e7e0ec80d0 100644 --- a/src/cmd/go/internal/modget/mvs.go +++ b/src/cmd/go/internal/modget/mvs.go @@ -145,7 +145,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.CheckAllowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, checkAllowedOrCurrent(m.Version)) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 8641cfec08..1c572d5d6d 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -35,6 +35,12 @@ type ImportMissingError struct { // and thus would be added by 'go mod tidy'. inAll bool + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string @@ -42,7 +48,7 @@ type ImportMissingError struct { func (e *ImportMissingError) Error() string { if e.Module.Path == "" { - if search.IsStandardImportPath(e.Path) { + if e.isStd { return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) } if e.QueryErr != nil { @@ -230,46 +236,67 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - return module.Version{}, "", &ImportMissingError{Path: path} + return module.Version{}, "", &ImportMissingError{Path: path, isStd: pathIsStd} } // queryImport attempts to locate a module that can be added to the current // build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. func queryImport(ctx context.Context, path string) (module.Version, error) { pathIsStd := search.IsStandardImportPath(path) - // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each - // module (golang.org/issue/26241). This should give a useful message - // in -mod=readonly, and it will allow us to add a requirement with -mod=mod. - if modFile != nil { - latest := map[string]string{} // path -> version - for _, r := range modFile.Replace { - if maybeInModule(path, r.Old.Path) { - // Don't use semver.Max here; need to preserve +incompatible suffix. - v := latest[r.Old.Path] - if semver.Compare(r.Old.Version, v) > 0 { - v = r.Old.Version + if cfg.BuildMod == "readonly" { + if pathIsStd { + // If the package would be in the standard library and none of the + // available replacement modules could concievably provide it, report it + // as a missing standard-library package instead of complaining that + // module lookups are disabled. + maybeReplaced := false + if index != nil { + for p := range index.highestReplaced { + if maybeInModule(path, p) { + maybeReplaced = true + break + } } - latest[r.Old.Path] = v } + if !maybeReplaced { + return module.Version{}, &ImportMissingError{Path: path, isStd: true} + } + } + + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + } - mods := make([]module.Version, 0, len(latest)) - for p, v := range latest { - // If the replacement didn't specify a version, synthesize a - // pseudo-version with an appropriate major version and a timestamp below - // any real timestamp. That way, if the main module is used from within - // some other module, the user will be able to upgrade the requirement to - // any real version they choose. - if v == "" { - if _, pathMajor, ok := module.SplitPathVersion(p); ok && len(pathMajor) > 0 { - v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + if index != nil { + var mods []module.Version + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue + } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") } else { - v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + mv = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") } } - mods = append(mods, module.Version{Path: p, Version: v}) + mods = append(mods, module.Version{Path: mp, Version: mv}) } // Every module path in mods is a prefix of the import path. @@ -310,17 +337,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // QueryPattern cannot possibly find a module containing this package. // // Instead of trying QueryPattern, report an ImportMissingError immediately. - return module.Version{}, &ImportMissingError{Path: path} - } - - if cfg.BuildMod == "readonly" { - var queryErr error - if cfg.BuildModExplicit { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else if cfg.BuildModReason != "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } - return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + return module.Version{}, &ImportMissingError{Path: path, isStd: true} } // Look up module containing the package, for addition to the build list. diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 6457a7d968..d15da892e6 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -35,13 +35,14 @@ var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile // at a specific point in time. type modFileIndex struct { - data []byte - dataNeedsFix bool // true if fixVersion applied a change while parsing data - module module.Version - goVersionV string // GoVersion with "v" prefix - require map[module.Version]requireMeta - replace map[module.Version]module.Version - exclude map[module.Version]bool + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersionV string // GoVersion with "v" prefix + require map[module.Version]requireMeta + replace map[module.Version]module.Version + highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements + exclude map[module.Version]bool } // index is the index of the go.mod file as of when it was last read or written. @@ -117,7 +118,7 @@ func checkRetractions(ctx context.Context, m module.Version) error { // v2.0.0+incompatible is not "latest" if v1.0.0 is current. rev, err := Query(ctx, path, "latest", findCurrentVersion(path), nil) if err != nil { - return &entry{err: err} + return &entry{nil, err} } // Load go.mod for that version. @@ -138,13 +139,19 @@ func checkRetractions(ctx context.Context, m module.Version) error { } summary, err := rawGoModSummary(rm) if err != nil { - return &entry{err: err} + return &entry{nil, err} } - return &entry{retract: summary.retract} + return &entry{summary.retract, nil} }).(*entry) - if e.err != nil { - return fmt.Errorf("loading module retractions: %v", e.err) + if err := e.err; err != nil { + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + var mErr *module.ModuleError + if errors.As(err, &mErr) { + err = mErr.Err + } + return &retractionLoadingError{m: m, err: err} } var rationale []string @@ -158,7 +165,7 @@ func checkRetractions(ctx context.Context, m module.Version) error { } } if isRetracted { - return &retractedError{rationale: rationale} + return module.VersionError(m, &retractedError{rationale: rationale}) } return nil } @@ -183,6 +190,19 @@ func (e *retractedError) Is(err error) bool { return err == ErrDisallowed } +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + // ShortRetractionRationale returns a retraction rationale string that is safe // to print in a terminal. It returns hard-coded strings if the rationale // is empty, too long, or contains non-printable characters. @@ -255,6 +275,14 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.replace[r.Old] = r.New } + i.highestReplaced = make(map[string]string) + for _, r := range modFile.Replace { + v, ok := i.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + i.highestReplaced[r.Old.Path] = r.Old.Version + } + } + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) for _, x := range modFile.Exclude { i.exclude[x.Mod] = true diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 44076fb615..6a3fd103fc 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -11,14 +11,15 @@ import ( "os" pathpkg "path" "path/filepath" + "sort" "strings" "sync" + "time" "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/trace" "golang.org/x/mod/module" @@ -106,136 +107,42 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed allowed = func(context.Context, module.Version) error { return nil } } - // Parse query to detect parse errors (and possibly handle query) - // before any network I/O. - badVersion := func(v string) (*modfetch.RevInfo, error) { - return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) - } - matchesMajor := func(v string) bool { - _, pathMajor, ok := module.SplitPathVersion(path) - if !ok { - return false - } - return module.CheckPathMajor(v, pathMajor) == nil - } - var ( - match = func(m module.Version) bool { return true } - - prefix string - preferOlder bool - mayUseLatest bool - preferIncompatible bool = strings.HasSuffix(current, "+incompatible") - ) - switch { - case query == "latest": - mayUseLatest = true - - case query == "upgrade": - mayUseLatest = true - - case query == "patch": - if current == "" { - mayUseLatest = true - } else { - prefix = semver.MajorMinor(current) - match = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) - } - } - - case strings.HasPrefix(query, "<="): - v := query[len("<="):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, "<"): - v := query[len("<"):] - if !semver.IsValid(v) { - return badVersion(v) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, ">="): - v := query[len(">="):] - if !semver.IsValid(v) { - return badVersion(v) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 + if path == Target.Path { + if query != "latest" { + return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } + return &modfetch.RevInfo{Version: Target.Version}, nil + } - case strings.HasPrefix(query, ">"): - v := query[len(">"):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } - case semver.IsValid(query) && isSemverPrefix(query): - match = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) - } - prefix = query + "." - if !matchesMajor(query) { - preferIncompatible = true - } + repo, err := lookupRepo(proxy, path) + if err != nil { + return nil, err + } - default: - // Direct lookup of semantic version or commit identifier. - - // If the query is a valid semantic version and that version is replaced, - // use the replacement module without searching the proxy. - canonicalQuery := module.CanonicalVersion(query) - if canonicalQuery != "" { - m := module.Version{Path: path, Version: query} - if r := Replacement(m); r.Path != "" { - if err := allowed(ctx, m); errors.Is(err, ErrDisallowed) { - return nil, err - } - return &modfetch.RevInfo{Version: query}, nil - } - } + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. - repo := modfetch.Lookup(proxy, path) info, err := repo.Stat(query) if err != nil { queryErr := err // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. + canonicalQuery := module.CanonicalVersion(query) if canonicalQuery != "" && query != canonicalQuery { info, err = repo.Stat(canonicalQuery) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -250,35 +157,16 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, err } return info, nil - } - - if path == Target.Path { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) - } - if err := allowed(ctx, Target); err != nil { - return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) - } - return &modfetch.RevInfo{Version: Target.Version}, nil - } - - if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") { - return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path) + } else if err != nil { + return nil, err } // Load versions and execute query. - repo := modfetch.Lookup(proxy, path) - versions, err := repo.Versions(prefix) + versions, err := repo.Versions(qm.prefix) if err != nil { return nil, err } - matchAndAllowed := func(ctx context.Context, m module.Version) error { - if !match(m) { - return ErrDisallowed - } - return allowed(ctx, m) - } - releases, prereleases, err := filterVersions(ctx, path, versions, matchAndAllowed, preferIncompatible) + releases, prereleases, err := qm.filterVersions(ctx, versions) if err != nil { return nil, err } @@ -289,11 +177,30 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, err } - // For "upgrade" and "patch", make sure we don't accidentally downgrade - // from a newer prerelease or from a chronologically newer pseudoversion. - if current != "" && (query == "upgrade" || query == "patch") { + if (query == "upgrade" || query == "patch") && modfetch.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) currentTime, err := modfetch.PseudoVersionTime(current) - if semver.Compare(rev.Version, current) < 0 || (err == nil && rev.Time.Before(currentTime)) { + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } return repo.Stat(current) } } @@ -301,7 +208,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return rev, nil } - if preferOlder { + if qm.preferLower { if len(releases) > 0 { return lookup(releases[0]) } @@ -317,13 +224,10 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } } - if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo - // if it is allowed. + if qm.mayUseLatest { latest, err := repo.Latest() if err == nil { - m := module.Version{Path: path, Version: latest.Version} - if err := allowed(ctx, m); !errors.Is(err, ErrDisallowed) { + if qm.allowsVersion(ctx, latest.Version) { return lookup(latest.Version) } } else if !errors.Is(err, os.ErrNotExist) { @@ -331,6 +235,14 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } } + if (query == "upgrade" || query == "patch") && current != "" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return lookup(current) + } + return nil, &NoMatchingVersionError{query: query, current: current} } @@ -368,10 +280,151 @@ func isSemverPrefix(v string) bool { return true } -// matchSemverPrefix reports whether the shortened semantic version p -// matches the full-width (non-shortened) semantic version v. -func matchSemverPrefix(p, v string) bool { - return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p && semver.Prerelease(v) == "" +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case query == "patch": + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.prefix = semver.MajorMinor(current) + "." + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case semver.IsValid(query): + if isSemverPrefix(query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return semver.Compare(mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return semver.Compare(mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + return false + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true } // filterVersions classifies versions into releases and pre-releases, filtering @@ -382,14 +435,32 @@ func matchSemverPrefix(p, v string) bool { // // If the allowed predicate returns an error not equivalent to ErrDisallowed, // filterVersions returns that error. -func filterVersions(ctx context.Context, path string, versions []string, allowed AllowedFunc, preferIncompatible bool) (releases, prereleases []string, err error) { +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + var lastCompatible string for _, v := range versions { - if err := allowed(ctx, module.Version{Path: path, Version: v}); errors.Is(err, ErrDisallowed) { + if !qm.allowsVersion(ctx, v) { continue } - if !preferIncompatible { + if !needIncompatible { + // We're not yet sure whether we need to include +incomptaible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + if !strings.HasSuffix(v, "+incompatible") { lastCompatible = v } else if lastCompatible != "" { @@ -397,19 +468,22 @@ func filterVersions(ctx context.Context, path string, versions []string, allowed // ignore any version with a higher (+incompatible) major version. (See // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - - ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) if err != nil { return nil, nil, err } if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. break } // No acceptable compatible release has a go.mod file, so the versioning // for the module might not be module-aware, and we should respect // legacy major-version tags. - preferIncompatible = true + needIncompatible = true } } @@ -766,3 +840,157 @@ func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { fi, err := os.Stat(filepath.Join(root, "go.mod")) return err == nil && !fi.IsDir(), nil } + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + Versions(prefix string) ([]string, error) + Stat(rev string) (*modfetch.RevInfo, error) + Latest() (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(proxy, path string) (repo versionRepo, err error) { + err = module.CheckPath(path) + if err == nil { + repo = modfetch.Lookup(proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if index == nil { + return repo, err + } + if _, ok := index.highestReplaced[path]; !ok { + return repo, err + } + + return &replacementRepo{repo: repo}, nil +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil } +func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err } +func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(prefix string) ([]string, error) { + versions, err := rr.repo.Versions(prefix) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + added := false + if index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !modfetch.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + + if !added { + return versions, nil + } + + sort.Slice(versions, func(i, j int) bool { + return semver.Compare(versions[i], versions[j]) < 0 + }) + uniq := versions[:1] + for _, v := range versions { + if v != uniq[len(uniq)-1] { + uniq = append(uniq, v) + } + } + return uniq, nil +} + +func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(rev) + if err == nil || index == nil || len(index.replace) == 0 { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest() + + if index != nil { + path := rr.ModulePath() + if v, ok := index.highestReplaced[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if modfetch.IsPseudoVersion(v) { + rev.Time, _ = modfetch.PseudoVersionTime(v) + rev.Short, _ = modfetch.PseudoVersionRev(v) + } + return rev, nil +} diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 351826f2ab..777a56b977 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -45,7 +45,7 @@ var ( queryRepoV3 = queryRepo + "/v3" // Empty version list (no semver tags), not actually empty. - emptyRepo = "vcs-test.golang.org/git/emptytest.git" + emptyRepoPath = "vcs-test.golang.org/git/emptytest.git" ) var queryTests = []struct { @@ -121,14 +121,14 @@ var queryTests = []struct { {path: queryRepo, query: "upgrade", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", current: "v0.0.0-20190513201126-42abcb6df8ee", vers: "v0.0.0-20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", allow: "NOMATCH", err: `no matching versions for query "upgrade"`}, - {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `no matching versions for query "upgrade" (current version is v1.9.9)`}, + {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `vcs-test.golang.org/git/querytest.git@v1.9.9: disallowed module version`}, {path: queryRepo, query: "upgrade", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: "patch", current: "", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v0.1.0", vers: "v0.1.2"}, {path: queryRepo, query: "patch", current: "v1.9.0", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, - {path: queryRepo, query: "patch", current: "v1.99.99", err: `no matching versions for query "patch" (current version is v1.99.99)`}, + {path: queryRepo, query: "patch", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: ">v1.9.9", vers: "v1.9.10-pre1"}, {path: queryRepo, query: ">v1.10.0", err: `no matching versions for query ">v1.10.0"`}, {path: queryRepo, query: ">=v1.10.0", err: `no matching versions for query ">=v1.10.0"`}, @@ -171,9 +171,9 @@ var queryTests = []struct { // That should prevent us from resolving any version for the /v3 path. {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, - {path: emptyRepo, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, - {path: emptyRepo, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, - {path: emptyRepo, query: "v0.0.0", err: `no matching versions for query ">v0.0.0"`}, + {path: emptyRepoPath, query: " Date: Mon, 12 Oct 2020 15:32:49 -0400 Subject: cmd/go/internal/modload: avoid using the global build list in QueryPattern The Query function allows the caller to specify the current version of the requested module, but the QueryPattern function is missing that parameter: instead, it always assumes that the current version is the one selected from the global build list. This change removes that assumption, instead adding a callback function to determine the current version. (The callback is currently invoked once per candidate module, regardless of whether that module exists, but in a future change we can refactor it to invoke the callback only when needed.) For #36460 For #40775 Change-Id: I001a4a8ab24f5b4fcc66a670d9bd305b47e948ba Reviewed-on: https://go-review.googlesource.com/c/go/+/261640 Trust: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/internal/modget/get.go | 2 +- src/cmd/go/internal/modload/buildlist.go | 15 +++++++++++++++ src/cmd/go/internal/modload/import.go | 2 +- src/cmd/go/internal/modload/modfile.go | 2 +- src/cmd/go/internal/modload/query.go | 15 +++------------ src/cmd/go/internal/work/build.go | 2 +- 6 files changed, 22 insertions(+), 16 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 9e2fb8e408..171c070ab3 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -912,7 +912,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Selected, allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 059b020420..95a68637c6 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -52,6 +52,21 @@ func LoadedModules() []module.Version { return buildList } +// Selected returns the selected version of the module with the given path, or +// the empty string if the given module has no selected version +// (either because it is not required or because it is the Target module). +func Selected(path string) (version string) { + if path == Target.Path { + return "" + } + for _, m := range buildList { + if m.Path == path { + return m.Version + } + } + return "" +} + // SetBuildList sets the module build list. // The caller is responsible for ensuring that the list is valid. // SetBuildList does not retain a reference to the original list. diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 1c572d5d6d..6d0d8de944 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -345,7 +345,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPattern(ctx, path, "latest", CheckAllowed) + candidates, err := QueryPattern(ctx, path, "latest", Selected, CheckAllowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index d15da892e6..006db4f169 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -116,7 +116,7 @@ func checkRetractions(ctx context.Context, m module.Version) error { // Ignore exclusions from the main module's go.mod. // We may need to account for the current version: for example, // v2.0.0+incompatible is not "latest" if v1.0.0 is current. - rev, err := Query(ctx, path, "latest", findCurrentVersion(path), nil) + rev, err := Query(ctx, path, "latest", Selected(path), nil) if err != nil { return &entry{nil, err} } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 6a3fd103fc..d16a247f72 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -516,7 +516,7 @@ type QueryResult struct { // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFunc) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) defer span.Done() @@ -591,9 +591,9 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFun ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) defer span.Done() - current := findCurrentVersion(path) + pathCurrent := current(path) r.Mod.Path = path - r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed) if err != nil { return r, err } @@ -649,15 +649,6 @@ func modulePrefixesExcludingTarget(path string) []string { return prefixes } -func findCurrentVersion(path string) string { - for _, m := range buildList { - if m.Path == path { - return m.Version - } - } - return "" -} - type prefixResult struct { QueryResult err error diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 21342ac8ba..3531612dc6 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -741,7 +741,7 @@ func installOutsideModule(ctx context.Context, args []string) { // Don't check for retractions if a specific revision is requested. allowed = nil } - qrs, err := modload.QueryPattern(ctx, patterns[0], version, allowed) + qrs, err := modload.QueryPattern(ctx, patterns[0], version, modload.Selected, allowed) if err != nil { base.Fatalf("go install %s: %v", args[0], err) } -- cgit v1.2.1 From 30119bcca997d154e4ab200b01afa7007b088994 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 16 Oct 2020 21:57:46 -0400 Subject: cmd/go/internal/modload: fix sort condition in (*replacementRepo).Versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In CL 258220 I added replacement versions to the repo versions used in the modload.Query functions. The versions are computed from a map in the modfile index, which has a nondeterministic iteration order. I added a short-circuit condition to skip sorting in the (vastly common) case where no replacement versions are added. However, while cleaning up the change I accidentally deleted the line of code that sets that condition. As a result, the test of that functionality (mod_get_replaced) has been failing nondeterministically. This change fixes the condition by comparing the slices before and after adding versions, rather than by setting a separate variable. The test now passes reliably (tested with -count=200). Updates #41577 Updates #41416 Updates #37438 Updates #26241 Change-Id: I49a66a3a5510da00ef42b47f20a168de66100db6 Reviewed-on: https://go-review.googlesource.com/c/go/+/263266 Trust: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Daniel Martí --- src/cmd/go/internal/modload/query.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index d16a247f72..3b27e66d01 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -891,12 +891,12 @@ func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } // Versions returns the versions from rr.repo augmented with any matching // replacement versions. func (rr *replacementRepo) Versions(prefix string) ([]string, error) { - versions, err := rr.repo.Versions(prefix) + repoVersions, err := rr.repo.Versions(prefix) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } - added := false + versions := repoVersions if index != nil && len(index.replace) > 0 { path := rr.ModulePath() for m, _ := range index.replace { @@ -906,7 +906,7 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) { } } - if !added { + if len(versions) == len(repoVersions) { // No replacement versions added. return versions, nil } -- cgit v1.2.1 From 4d1cecdee8360ef12a817c124d7a04c9d29741c3 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 14 Oct 2020 08:06:54 -0400 Subject: cmd/dist,cmd/go: broaden use of asm macro GOEXPERIMENT_REGABI This extends a change made in https://golang.org/cl/252258 to the go command (to define an asm macro when GOEXPERIMENT=regabi is in effect); we need this same macro during the bootstrap build in order to build the runtime correctly. In addition, expand the set of packages where the macro is applied to {runtime, reflect, syscall, runtime/internal/*}, and move the logic for deciding when something is a "runtime package" out of the assembler and into cmd/{go,dist}, introducing a new assembler command line flag instead. Updates #27539, #40724. Change-Id: Ifcc7f029f56873584de1e543c55b0d3e54ad6c49 Reviewed-on: https://go-review.googlesource.com/c/go/+/262317 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/go/internal/work/gc.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 56ad1872be..2df4a52ba5 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -292,14 +292,20 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } - if p.ImportPath == "runtime" && objabi.Regabi_enabled != 0 { - // In order to make it easier to port runtime assembly - // to the register ABI, we introduce a macro - // indicating the experiment is enabled. - // - // TODO(austin): Remove this once we commit to the - // register ABI (#40724). - args = append(args, "-D=GOEXPERIMENT_REGABI=1") + if objabi.IsRuntimePackagePath(pkgpath) { + args = append(args, "-compilingRuntime") + if objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // Note: a similar change also appears in + // cmd/dist/build.go. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { -- cgit v1.2.1 From 7bb721b9384bdd196befeaed593b185f7f2a5589 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 7 Jul 2020 13:49:21 -0400 Subject: all: update references to symbols moved from os to io/fs The old os references are still valid, but update our code to reflect best practices and get used to the new locations. Code compiled with the bootstrap toolchain (cmd/asm, cmd/dist, cmd/compile, debug/elf) must remain Go 1.4-compatible and is excluded. For #41190. Change-Id: I8f9526977867c10a221e2f392f78d7dec073f1bd Reviewed-on: https://go-review.googlesource.com/c/go/+/243907 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Rob Pike --- src/cmd/go/internal/cache/cache.go | 3 +- src/cmd/go/internal/fsys/fsys.go | 41 +++++++++++----------- src/cmd/go/internal/fsys/fsys_test.go | 39 ++++++++++---------- src/cmd/go/internal/imports/scan.go | 3 +- src/cmd/go/internal/load/pkg.go | 5 +-- .../lockedfile/internal/filelock/filelock.go | 5 +-- .../lockedfile/internal/filelock/filelock_fcntl.go | 6 ++-- .../lockedfile/internal/filelock/filelock_other.go | 6 ++-- .../lockedfile/internal/filelock/filelock_plan9.go | 8 ++--- .../lockedfile/internal/filelock/filelock_unix.go | 4 +-- .../internal/filelock/filelock_windows.go | 6 ++-- src/cmd/go/internal/lockedfile/lockedfile.go | 9 ++--- .../go/internal/lockedfile/lockedfile_filelock.go | 3 +- src/cmd/go/internal/lockedfile/lockedfile_plan9.go | 9 ++--- src/cmd/go/internal/modcmd/vendor.go | 7 ++-- src/cmd/go/internal/modcmd/verify.go | 9 ++--- src/cmd/go/internal/modfetch/cache.go | 7 ++-- src/cmd/go/internal/modfetch/codehost/codehost.go | 9 ++--- src/cmd/go/internal/modfetch/codehost/git.go | 15 ++++---- src/cmd/go/internal/modfetch/codehost/git_test.go | 3 +- src/cmd/go/internal/modfetch/codehost/vcs.go | 3 +- src/cmd/go/internal/modfetch/coderepo.go | 7 ++-- src/cmd/go/internal/modfetch/fetch.go | 11 +++--- src/cmd/go/internal/modfetch/proxy.go | 8 ++--- src/cmd/go/internal/modfetch/repo.go | 5 +-- src/cmd/go/internal/modfetch/sumdb.go | 7 ++-- src/cmd/go/internal/modload/import.go | 3 +- src/cmd/go/internal/modload/load.go | 3 +- src/cmd/go/internal/modload/query.go | 11 +++--- src/cmd/go/internal/modload/search.go | 5 +-- src/cmd/go/internal/modload/stat_openfile.go | 3 +- src/cmd/go/internal/modload/stat_unix.go | 3 +- src/cmd/go/internal/modload/stat_windows.go | 6 ++-- src/cmd/go/internal/modload/vendor.go | 4 +-- src/cmd/go/internal/renameio/renameio.go | 7 ++-- src/cmd/go/internal/renameio/umask_test.go | 5 +-- src/cmd/go/internal/search/search.go | 7 ++-- src/cmd/go/internal/test/test.go | 3 +- src/cmd/go/internal/vcs/vcs.go | 5 +-- src/cmd/go/internal/version/version.go | 11 +++--- src/cmd/go/internal/web/api.go | 6 ++-- src/cmd/go/internal/web/file_test.go | 5 +-- src/cmd/go/internal/work/build_test.go | 3 +- src/cmd/go/internal/work/exec.go | 9 ++--- 44 files changed, 184 insertions(+), 153 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/cache/cache.go b/src/cmd/go/internal/cache/cache.go index 15545ac31f..5464fe5685 100644 --- a/src/cmd/go/internal/cache/cache.go +++ b/src/cmd/go/internal/cache/cache.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -54,7 +55,7 @@ func Open(dir string) (*Cache, error) { return nil, err } if !info.IsDir() { - return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 489af93496..67359ffb6d 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -240,7 +241,7 @@ var errNotDir = errors.New("not a directory") // readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. // Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory // can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). -func readDir(dir string) ([]os.FileInfo, error) { +func readDir(dir string) ([]fs.FileInfo, error) { fis, err := ioutil.ReadDir(dir) if err == nil { return fis, nil @@ -249,25 +250,25 @@ func readDir(dir string) ([]os.FileInfo, error) { if os.IsNotExist(err) { return nil, err } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} } else { return nil, err } } -// ReadDir provides a slice of os.FileInfo entries corresponding +// ReadDir provides a slice of fs.FileInfo entries corresponding // to the overlaid files in the directory. -func ReadDir(dir string) ([]os.FileInfo, error) { +func ReadDir(dir string) ([]fs.FileInfo, error) { dir = canonicalize(dir) if _, ok := parentIsOverlayFile(dir); ok { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} } dirNode := overlay[dir] if dirNode == nil { return readDir(dir) } else if dirNode.isDeleted() { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} } diskfis, err := readDir(dir) if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { @@ -275,7 +276,7 @@ func ReadDir(dir string) ([]os.FileInfo, error) { } // Stat files in overlay to make composite list of fileinfos - files := make(map[string]os.FileInfo) + files := make(map[string]fs.FileInfo) for _, f := range diskfis { files[f.Name()] = f } @@ -327,14 +328,14 @@ func Open(path string) (*os.File, error) { cpath := canonicalize(path) if node, ok := overlay[cpath]; ok { if node.isDir() { - return nil, &os.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} + return nil, &fs.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} } return os.Open(node.actualFilePath) } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { // The file is deleted explicitly in the Replace map, // or implicitly because one of its parent directories was // replaced by a file. - return nil, &os.PathError{ + return nil, &fs.PathError{ Op: "Open", Path: path, Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} @@ -387,7 +388,7 @@ func IsDirWithGoFiles(dir string) (bool, error) { // walk recursively descends path, calling walkFn. Copied, with some // modifications from path/filepath.walk. -func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { +func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { if !info.IsDir() { return walkFn(path, info, nil) } @@ -432,11 +433,11 @@ func Walk(root string, walkFn filepath.WalkFunc) error { } // lstat implements a version of os.Lstat that operates on the overlay filesystem. -func lstat(path string) (os.FileInfo, error) { +func lstat(path string) (fs.FileInfo, error) { cpath := canonicalize(path) if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { - return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} } node, ok := overlay[cpath] @@ -447,7 +448,7 @@ func lstat(path string) (os.FileInfo, error) { switch { case node.isDeleted(): - return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} case node.isDir(): return fakeDir(filepath.Base(cpath)), nil default: @@ -459,22 +460,22 @@ func lstat(path string) (os.FileInfo, error) { } } -// fakeFile provides an os.FileInfo implementation for an overlaid file, +// fakeFile provides an fs.FileInfo implementation for an overlaid file, // so that the file has the name of the overlaid file, but takes all // other characteristics of the replacement file. type fakeFile struct { name string - real os.FileInfo + real fs.FileInfo } func (f fakeFile) Name() string { return f.name } func (f fakeFile) Size() int64 { return f.real.Size() } -func (f fakeFile) Mode() os.FileMode { return f.real.Mode() } +func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } func (f fakeFile) IsDir() bool { return f.real.IsDir() } func (f fakeFile) Sys() interface{} { return f.real.Sys() } -// missingFile provides an os.FileInfo for an overlaid file where the +// missingFile provides an fs.FileInfo for an overlaid file where the // destination file in the overlay doesn't exist. It returns zero values // for the fileInfo methods other than Name, set to the file's name, and Mode // set to ModeIrregular. @@ -482,19 +483,19 @@ type missingFile string func (f missingFile) Name() string { return string(f) } func (f missingFile) Size() int64 { return 0 } -func (f missingFile) Mode() os.FileMode { return os.ModeIrregular } +func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } func (f missingFile) IsDir() bool { return false } func (f missingFile) Sys() interface{} { return nil } -// fakeDir provides an os.FileInfo implementation for directories that are +// fakeDir provides an fs.FileInfo implementation for directories that are // implicitly created by overlaid files. Each directory in the // path of an overlaid file is considered to exist in the overlay filesystem. type fakeDir string func (f fakeDir) Name() string { return string(f) } func (f fakeDir) Size() int64 { return 0 } -func (f fakeDir) Mode() os.FileMode { return os.ModeDir | 0500 } +func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } func (f fakeDir) IsDir() bool { return true } func (f fakeDir) Sys() interface{} { return nil } diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 6cf59fba47..ba9f05d00b 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "internal/testenv" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -291,8 +292,8 @@ x _, gotErr := ReadDir(dir) if gotErr == nil { t.Errorf("ReadDir(%q): got no error, want error", dir) - } else if _, ok := gotErr.(*os.PathError); !ok { - t.Errorf("ReadDir(%q): got error with string %q and type %T, want os.PathError", dir, gotErr.Error(), gotErr) + } else if _, ok := gotErr.(*fs.PathError); !ok { + t.Errorf("ReadDir(%q): got error with string %q and type %T, want fs.PathError", dir, gotErr.Error(), gotErr) } } } @@ -489,7 +490,7 @@ func TestWalk(t *testing.T) { path string name string size int64 - mode os.FileMode + mode fs.FileMode isDir bool } testCases := []struct { @@ -504,7 +505,7 @@ func TestWalk(t *testing.T) { `, ".", []file{ - {".", "root", 0, os.ModeDir | 0700, true}, + {".", "root", 0, fs.ModeDir | 0700, true}, {"file.txt", "file.txt", 0, 0600, false}, }, }, @@ -520,7 +521,7 @@ contents of other file `, ".", []file{ - {".", "root", 0, os.ModeDir | 0500, true}, + {".", "root", 0, fs.ModeDir | 0500, true}, {"file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, }, @@ -536,7 +537,7 @@ contents of other file `, ".", []file{ - {".", "root", 0, os.ModeDir | 0500, true}, + {".", "root", 0, fs.ModeDir | 0500, true}, {"file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, }, @@ -552,8 +553,8 @@ contents of other file `, ".", []file{ - {".", "root", 0, os.ModeDir | 0500, true}, - {"dir", "dir", 0, os.ModeDir | 0500, true}, + {".", "root", 0, fs.ModeDir | 0500, true}, + {"dir", "dir", 0, fs.ModeDir | 0500, true}, {"dir" + string(filepath.Separator) + "file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, }, @@ -565,7 +566,7 @@ contents of other file initOverlay(t, tc.overlay) var got []file - Walk(tc.root, func(path string, info os.FileInfo, err error) error { + Walk(tc.root, func(path string, info fs.FileInfo, err error) error { got = append(got, file{path, info.Name(), info.Size(), info.Mode(), info.IsDir()}) return nil }) @@ -580,8 +581,8 @@ contents of other file if got[i].name != tc.wantFiles[i].name { t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name) } - if got[i].mode&(os.ModeDir|0700) != tc.wantFiles[i].mode { - t.Errorf("mode&(os.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(os.ModeDir|0700), tc.wantFiles[i].mode) + if got[i].mode&(fs.ModeDir|0700) != tc.wantFiles[i].mode { + t.Errorf("mode&(fs.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(fs.ModeDir|0700), tc.wantFiles[i].mode) } if got[i].isDir != tc.wantFiles[i].isDir { t.Errorf("isDir for file #%v in walk, got %v, want %v", i, got[i].isDir, tc.wantFiles[i].isDir) @@ -610,7 +611,7 @@ func TestWalk_SkipDir(t *testing.T) { `) var seen []string - Walk(".", func(path string, info os.FileInfo, err error) error { + Walk(".", func(path string, info fs.FileInfo, err error) error { seen = append(seen, path) if path == "skipthisdir" || path == filepath.Join("dontskip", "skip") { return filepath.SkipDir @@ -635,7 +636,7 @@ func TestWalk_Error(t *testing.T) { initOverlay(t, "{}") alreadyCalled := false - err := Walk("foo", func(path string, info os.FileInfo, err error) error { + err := Walk("foo", func(path string, info fs.FileInfo, err error) error { if alreadyCalled { t.Fatal("expected walk function to be called exactly once, but it was called more than once") } @@ -683,7 +684,7 @@ func TestWalk_Symlink(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var got []string - err := Walk(tc.dir, func(path string, info os.FileInfo, err error) error { + err := Walk(tc.dir, func(path string, info fs.FileInfo, err error) error { got = append(got, path) if err != nil { t.Errorf("walkfn: got non nil err argument: %v, want nil err argument", err) @@ -706,7 +707,7 @@ func TestLstat(t *testing.T) { type file struct { name string size int64 - mode os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions + mode fs.FileMode // mode & (fs.ModeDir|0x700): only check 'user' permissions isDir bool } @@ -771,7 +772,7 @@ contents`, -- dir/foo.txt -- `, "dir", - file{"dir", 0, 0700 | os.ModeDir, true}, + file{"dir", 0, 0700 | fs.ModeDir, true}, false, }, { @@ -780,7 +781,7 @@ contents`, -- dummy.txt -- `, "dir", - file{"dir", 0, 0500 | os.ModeDir, true}, + file{"dir", 0, 0500 | fs.ModeDir, true}, false, }, } @@ -801,8 +802,8 @@ contents`, if got.Name() != tc.want.name { t.Errorf("lstat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) } - if got.Mode()&(os.ModeDir|0700) != tc.want.mode { - t.Errorf("lstat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode) + if got.Mode()&(fs.ModeDir|0700) != tc.want.mode { + t.Errorf("lstat(%q).Mode()&(fs.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(fs.ModeDir|0700), tc.want.mode) } if got.IsDir() != tc.want.isDir { t.Errorf("lstat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) diff --git a/src/cmd/go/internal/imports/scan.go b/src/cmd/go/internal/imports/scan.go index 42ee49aaaa..d45393f36c 100644 --- a/src/cmd/go/internal/imports/scan.go +++ b/src/cmd/go/internal/imports/scan.go @@ -6,6 +6,7 @@ package imports import ( "fmt" + "io/fs" "os" "path/filepath" "sort" @@ -26,7 +27,7 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. - if info.Mode()&os.ModeSymlink != 0 { + if info.Mode()&fs.ModeSymlink != 0 { info, err = os.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index f73b79d089..066ff6c981 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -14,6 +14,7 @@ import ( "go/build" "go/scanner" "go/token" + "io/fs" "io/ioutil" "os" pathpkg "path" @@ -2300,7 +2301,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { // to make it look like this is a standard package or // command directory. So that local imports resolve // consistently, the files must all be in the same directory. - var dirent []os.FileInfo + var dirent []fs.FileInfo var dir string for _, file := range gofiles { fi, err := os.Stat(file) @@ -2321,7 +2322,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { } dirent = append(dirent, fi) } - ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } + ctxt.ReadDir = func(string) ([]fs.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { modload.ImportFromFiles(ctx, gofiles) diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go index aba3eed776..05f27c321a 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go @@ -9,6 +9,7 @@ package filelock import ( "errors" + "io/fs" "os" ) @@ -24,7 +25,7 @@ type File interface { Fd() uintptr // Stat returns the FileInfo structure describing file. - Stat() (os.FileInfo, error) + Stat() (fs.FileInfo, error) } // Lock places an advisory write lock on the file, blocking until it can be @@ -87,7 +88,7 @@ var ErrNotSupported = errors.New("operation not supported") // underlyingError returns the underlying error for known os error types. func underlyingError(err error) error { switch err := err.(type) { - case *os.PathError: + case *fs.PathError: return err.Err case *os.LinkError: return err.Err diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go index 8776c5741c..1fa4327a89 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -18,8 +18,8 @@ package filelock import ( "errors" "io" + "io/fs" "math/rand" - "os" "sync" "syscall" "time" @@ -61,7 +61,7 @@ func lock(f File, lt lockType) (err error) { mu.Lock() if i, dup := inodes[f]; dup && i != ino { mu.Unlock() - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: errors.New("inode for file changed since last Lock or RLock"), @@ -152,7 +152,7 @@ func lock(f File, lt lockType) (err error) { if err != nil { unlock(f) - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go index 107611e1ce..bc480343fc 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go @@ -6,7 +6,7 @@ package filelock -import "os" +import "io/fs" type lockType int8 @@ -16,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -24,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go index afdffe323f..0798ee469a 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go @@ -6,9 +6,7 @@ package filelock -import ( - "os" -) +import "io/fs" type lockType int8 @@ -18,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -26,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go index 78f2c51129..ed07bac608 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go @@ -7,7 +7,7 @@ package filelock import ( - "os" + "io/fs" "syscall" ) @@ -26,7 +26,7 @@ func lock(f File, lt lockType) (err error) { } } if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go index 43e85e450e..19de27eb9b 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go @@ -8,7 +8,7 @@ package filelock import ( "internal/syscall/windows" - "os" + "io/fs" "syscall" ) @@ -34,7 +34,7 @@ func lock(f File, lt lockType) error { err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, @@ -47,7 +47,7 @@ func unlock(f File) error { ol := new(syscall.Overlapped) err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/lockedfile.go b/src/cmd/go/internal/lockedfile/lockedfile.go index 59b2dba44c..503024da4b 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile.go +++ b/src/cmd/go/internal/lockedfile/lockedfile.go @@ -9,6 +9,7 @@ package lockedfile import ( "fmt" "io" + "io/fs" "io/ioutil" "os" "runtime" @@ -35,7 +36,7 @@ type osFile struct { // OpenFile is like os.OpenFile, but returns a locked file. // If flag includes os.O_WRONLY or os.O_RDWR, the file is write-locked; // otherwise, it is read-locked. -func OpenFile(name string, flag int, perm os.FileMode) (*File, error) { +func OpenFile(name string, flag int, perm fs.FileMode) (*File, error) { var ( f = new(File) err error @@ -82,10 +83,10 @@ func Edit(name string) (*File, error) { // non-nil error. func (f *File) Close() error { if f.closed { - return &os.PathError{ + return &fs.PathError{ Op: "close", Path: f.Name(), - Err: os.ErrClosed, + Err: fs.ErrClosed, } } f.closed = true @@ -108,7 +109,7 @@ func Read(name string) ([]byte, error) { // Write opens the named file (creating it with the given permissions if needed), // then write-locks it and overwrites it with the given content. -func Write(name string, content io.Reader, perm os.FileMode) (err error) { +func Write(name string, content io.Reader, perm fs.FileMode) (err error) { f, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err diff --git a/src/cmd/go/internal/lockedfile/lockedfile_filelock.go b/src/cmd/go/internal/lockedfile/lockedfile_filelock.go index f63dd8664b..10e1240efd 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile_filelock.go +++ b/src/cmd/go/internal/lockedfile/lockedfile_filelock.go @@ -7,12 +7,13 @@ package lockedfile import ( + "io/fs" "os" "cmd/go/internal/lockedfile/internal/filelock" ) -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // On BSD systems, we could add the O_SHLOCK or O_EXLOCK flag to the OpenFile // call instead of locking separately, but we have to support separate locking // calls for Linux and Windows anyway, so it's simpler to use that approach diff --git a/src/cmd/go/internal/lockedfile/lockedfile_plan9.go b/src/cmd/go/internal/lockedfile/lockedfile_plan9.go index 4a52c94976..51681381d7 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile_plan9.go +++ b/src/cmd/go/internal/lockedfile/lockedfile_plan9.go @@ -7,6 +7,7 @@ package lockedfile import ( + "io/fs" "math/rand" "os" "strings" @@ -41,7 +42,7 @@ func isLocked(err error) bool { return false } -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // Plan 9 uses a mode bit instead of explicit lock/unlock syscalls. // // Per http://man.cat-v.org/plan_9/5/stat: “Exclusive use files may be open @@ -56,8 +57,8 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { // have the ModeExclusive bit set. Set it before we call OpenFile, so that we // can be confident that a successful OpenFile implies exclusive use. if fi, err := os.Stat(name); err == nil { - if fi.Mode()&os.ModeExclusive == 0 { - if err := os.Chmod(name, fi.Mode()|os.ModeExclusive); err != nil { + if fi.Mode()&fs.ModeExclusive == 0 { + if err := os.Chmod(name, fi.Mode()|fs.ModeExclusive); err != nil { return nil, err } } @@ -68,7 +69,7 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { nextSleep := 1 * time.Millisecond const maxSleep = 500 * time.Millisecond for { - f, err := os.OpenFile(name, flag, perm|os.ModeExclusive) + f, err := os.OpenFile(name, flag, perm|fs.ModeExclusive) if err == nil { return f, nil } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 1bc4ab3def..1b9ce60529 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -232,7 +233,7 @@ var metaPrefixes = []string{ } // matchMetadata reports whether info is a metadata file. -func matchMetadata(dir string, info os.FileInfo) bool { +func matchMetadata(dir string, info fs.FileInfo) bool { name := info.Name() for _, p := range metaPrefixes { if strings.HasPrefix(name, p) { @@ -243,7 +244,7 @@ func matchMetadata(dir string, info os.FileInfo) bool { } // matchPotentialSourceFile reports whether info may be relevant to a build operation. -func matchPotentialSourceFile(dir string, info os.FileInfo) bool { +func matchPotentialSourceFile(dir string, info fs.FileInfo) bool { if strings.HasSuffix(info.Name(), "_test.go") { return false } @@ -269,7 +270,7 @@ func matchPotentialSourceFile(dir string, info os.FileInfo) bool { } // copyDir copies all regular files satisfying match(info) from src to dst. -func copyDir(dst, src string, match func(dir string, info os.FileInfo) bool) { +func copyDir(dst, src string, match func(dir string, info fs.FileInfo) bool) { files, err := ioutil.ReadDir(src) if err != nil { base.Fatalf("go mod vendor: %v", err) diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index bd591d3f32..ce24793929 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "io/fs" "io/ioutil" "os" "runtime" @@ -88,8 +89,8 @@ func verifyMod(mod module.Version) []error { dir, dirErr := modfetch.DownloadDir(mod) data, err := ioutil.ReadFile(zip + "hash") if err != nil { - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) && - dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) && + dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // Nothing downloaded yet. Nothing to verify. return nil } @@ -98,7 +99,7 @@ func verifyMod(mod module.Version) []error { } h := string(bytes.TrimSpace(data)) - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) { // ok } else { hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) @@ -109,7 +110,7 @@ func verifyMod(mod module.Version) []error { errs = append(errs, fmt.Errorf("%s %s: zip has been modified (%v)", mod.Path, mod.Version, zip)) } } - if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // ok } else { hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go index 6eadb026c9..b7aa670250 100644 --- a/src/cmd/go/internal/modfetch/cache.go +++ b/src/cmd/go/internal/modfetch/cache.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -60,7 +61,7 @@ func CachePath(m module.Version, suffix string) (string, error) { // DownloadDir returns the directory to which m should have been downloaded. // An error will be returned if the module path or version cannot be escaped. -// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned // along with the directory if the directory does not exist or if the directory // is not completely populated. func DownloadDir(m module.Version) (string, error) { @@ -107,14 +108,14 @@ func DownloadDir(m module.Version) (string, error) { // DownloadDirPartialError is returned by DownloadDir if a module directory // exists but was not completely populated. // -// DownloadDirPartialError is equivalent to os.ErrNotExist. +// DownloadDirPartialError is equivalent to fs.ErrNotExist. type DownloadDirPartialError struct { Dir string Err error } func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } -func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } +func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist } // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go index df4cfdab1a..c5fbb31b2b 100644 --- a/src/cmd/go/internal/modfetch/codehost/codehost.go +++ b/src/cmd/go/internal/modfetch/codehost/codehost.go @@ -11,6 +11,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/fs" "io/ioutil" "os" "os/exec" @@ -105,7 +106,7 @@ type FileRev struct { Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev } -// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a +// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a // revision rather than a file. type UnknownRevisionError struct { Rev string @@ -115,10 +116,10 @@ func (e *UnknownRevisionError) Error() string { return "unknown revision " + e.Rev } func (UnknownRevisionError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } -// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given +// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given // repository or module contains no commits. var ErrNoCommits error = noCommitsError{} @@ -128,7 +129,7 @@ func (noCommitsError) Error() string { return "no commits" } func (noCommitsError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go index 5a35829c98..58b4b2f2d3 100644 --- a/src/cmd/go/internal/modfetch/codehost/git.go +++ b/src/cmd/go/internal/modfetch/codehost/git.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "net/url" "os" @@ -34,13 +35,13 @@ func LocalGitRepo(remote string) (Repo, error) { } // A notExistError wraps another error to retain its original text -// but makes it opaquely equivalent to os.ErrNotExist. +// but makes it opaquely equivalent to fs.ErrNotExist. type notExistError struct { err error } func (e notExistError) Error() string { return e.err.Error() } -func (notExistError) Is(err error) bool { return err == os.ErrNotExist } +func (notExistError) Is(err error) bool { return err == fs.ErrNotExist } const gitWorkDirType = "git3" @@ -188,7 +189,7 @@ func (r *gitRepo) loadRefs() { // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL // ourselves and see what code it serves. if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { - if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) { + if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { gitErr = notExistError{gitErr} } } @@ -505,7 +506,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { } out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -629,9 +630,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F case "tag", "commit": switch fileType { default: - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} case "missing": - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist} case "blob": f.Data = fileData } @@ -826,7 +827,7 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args) if err != nil { if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return nil, err } diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go index ba27c70f5a..16908b3e84 100644 --- a/src/cmd/go/internal/modfetch/codehost/git_test.go +++ b/src/cmd/go/internal/modfetch/codehost/git_test.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "internal/testenv" + "io/fs" "io/ioutil" "log" "os" @@ -210,7 +211,7 @@ var readFileTests = []struct { repo: gitrepo1, rev: "v2.3.4", file: "another.txt", - err: os.ErrNotExist.Error(), + err: fs.ErrNotExist.Error(), }, } diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go index 6278cb21e1..ec97fc7e1b 100644 --- a/src/cmd/go/internal/modfetch/codehost/vcs.go +++ b/src/cmd/go/internal/modfetch/codehost/vcs.go @@ -9,6 +9,7 @@ import ( "fmt" "internal/lazyregexp" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -377,7 +378,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go index d99a31d360..7f44e18a70 100644 --- a/src/cmd/go/internal/modfetch/coderepo.go +++ b/src/cmd/go/internal/modfetch/coderepo.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path" @@ -1040,7 +1041,7 @@ type zipFile struct { } func (f zipFile) Path() string { return f.name } -func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil } +func (f zipFile) Lstat() (fs.FileInfo, error) { return f.f.FileInfo(), nil } func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() } type dataFile struct { @@ -1049,7 +1050,7 @@ type dataFile struct { } func (f dataFile) Path() string { return f.name } -func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil } +func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil } func (f dataFile) Open() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(f.data)), nil } @@ -1060,7 +1061,7 @@ type dataFileInfo struct { func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) } func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } -func (fi dataFileInfo) Mode() os.FileMode { return 0644 } +func (fi dataFileInfo) Mode() fs.FileMode { return 0644 } func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } func (fi dataFileInfo) IsDir() bool { return false } func (fi dataFileInfo) Sys() interface{} { return nil } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 599419977a..6ff455e89c 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -67,7 +68,7 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { if err == nil { // The directory has already been completely extracted (no .partial file exists). return dir, nil - } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + } else if dir == "" || !errors.Is(err, fs.ErrNotExist) { return "", err } @@ -314,10 +315,10 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e func makeDirsReadOnly(dir string) { type pathMode struct { path string - mode os.FileMode + mode fs.FileMode } var dirs []pathMode // in lexical order - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err == nil && info.Mode()&0222 != 0 { if info.IsDir() { dirs = append(dirs, pathMode{path, info.Mode()}) @@ -336,7 +337,7 @@ func makeDirsReadOnly(dir string) { // any permission changes needed to do so. func RemoveAll(dir string) error { // Module cache has 0555 directories; make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { return nil // ignore errors walking in file system } @@ -441,7 +442,7 @@ func checkMod(mod module.Version) { } data, err := renameio.ReadFile(ziphash) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. return } diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 4ac26650a9..819990b403 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -9,9 +9,9 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "net/url" - "os" "path" pathpkg "path" "path/filepath" @@ -186,7 +186,7 @@ func proxyList() ([]proxySpec, error) { // TryProxies iterates f over each configured proxy (including "noproxy" and // "direct" if applicable) until f returns no error or until f returns an -// error that is not equivalent to os.ErrNotExist on a proxy configured +// error that is not equivalent to fs.ErrNotExist on a proxy configured // not to fall back on errors. // // TryProxies then returns that final error. @@ -222,7 +222,7 @@ func TryProxies(f func(proxy string) error) error { if err == nil { return nil } - isNotExistErr := errors.Is(err, os.ErrNotExist) + isNotExistErr := errors.Is(err, fs.ErrNotExist) if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) { bestErr = err @@ -428,7 +428,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { func (p *proxyRepo) Latest() (*RevInfo, error) { data, err := p.getBytes("@latest") if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { return nil, p.versionError("", err) } return p.latest() diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index c7cf5595bd..af9e24cefd 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -7,6 +7,7 @@ package modfetch import ( "fmt" "io" + "io/fs" "os" "sort" "strconv" @@ -432,7 +433,7 @@ func (r errRepo) Latest() (*RevInfo, error) { return nil func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } -// A notExistError is like os.ErrNotExist, but with a custom message +// A notExistError is like fs.ErrNotExist, but with a custom message type notExistError struct { err error } @@ -446,7 +447,7 @@ func (e notExistError) Error() string { } func (notExistError) Is(target error) bool { - return target == os.ErrNotExist + return target == fs.ErrNotExist } func (e notExistError) Unwrap() error { diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 47a2571531..5108961a33 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -12,6 +12,7 @@ import ( "bytes" "errors" "fmt" + "io/fs" "io/ioutil" "net/url" "os" @@ -182,7 +183,7 @@ func (c *dbClient) initBase() { return nil } }) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // No proxies, or all proxies failed (with 404, 410, or were were allowed // to fall back), or we reached an explicit "direct" or "off". c.base = c.direct @@ -203,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) { } targ := filepath.Join(cfg.SumdbDir, file) data, err = lockedfile.Read(targ) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Treat non-existent as empty, to bootstrap the "latest" file // the first time we connect to a given database. return []byte{}, nil @@ -257,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) { // during which the empty file can be locked for reading. // Treat observing an empty file as file not found. if err == nil && len(data) == 0 { - err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist} + err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist} } return data, err } diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 6d0d8de944..bcbc9b0c3a 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -10,6 +10,7 @@ import ( "fmt" "go/build" "internal/goroot" + "io/fs" "os" "path/filepath" "sort" @@ -347,7 +348,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { candidates, err := QueryPattern(ctx, path, "latest", Selected, CheckAllowed) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever // low-level error QueryPattern produced. return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 4ddb817cf1..4b3ded8326 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -98,6 +98,7 @@ import ( "errors" "fmt" "go/build" + "io/fs" "os" "path" pathpkg "path" @@ -364,7 +365,7 @@ func resolveLocalPackage(dir string) (string, error) { if os.IsNotExist(err) { // Canonicalize OS-specific errors to errDirectoryNotFound so that error // messages will be easier for users to search for. - return "", &os.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} } return "", err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 3b27e66d01..6b14768388 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "io/fs" "os" pathpkg "path" "path/filepath" @@ -145,7 +146,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed canonicalQuery := module.CanonicalVersion(query) if canonicalQuery != "" && query != canonicalQuery { info, err = repo.Stat(canonicalQuery) - if err != nil && !errors.Is(err, os.ErrNotExist) { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return info, err } } @@ -230,7 +231,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if qm.allowsVersion(ctx, latest.Version) { return lookup(latest.Version) } - } else if !errors.Is(err, os.ErrNotExist) { + } else if !errors.Is(err, fs.ErrNotExist) { return nil, err } } @@ -701,7 +702,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod noVersion = rErr } default: - if errors.Is(rErr, os.ErrNotExist) { + if errors.Is(rErr, fs.ErrNotExist) { if notExistErr == nil { notExistErr = rErr } @@ -744,7 +745,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod // A NoMatchingVersionError indicates that Query found a module at the requested // path, but not at any versions satisfying the query string and allow-function. // -// NOTE: NoMatchingVersionError MUST NOT implement Is(os.ErrNotExist). +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -765,7 +766,7 @@ func (e *NoMatchingVersionError) Error() string { // module at the requested version, but that module did not contain any packages // matching the requested pattern. // -// NOTE: PackageNotInModuleError MUST NOT implement Is(os.ErrNotExist). +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index 0f82026732..19289ceb9c 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -7,6 +7,7 @@ package modload import ( "context" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -54,7 +55,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { m.AddError(err) return nil @@ -85,7 +86,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { + if fi.Mode()&fs.ModeSymlink != 0 && want { if target, err := os.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } diff --git a/src/cmd/go/internal/modload/stat_openfile.go b/src/cmd/go/internal/modload/stat_openfile.go index 931aaf1577..7cdeaf47a2 100644 --- a/src/cmd/go/internal/modload/stat_openfile.go +++ b/src/cmd/go/internal/modload/stat_openfile.go @@ -13,12 +13,13 @@ package modload import ( + "io/fs" "os" ) // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(path string, _ os.FileInfo) bool { +func hasWritePerm(path string, _ fs.FileInfo) bool { if f, err := os.OpenFile(path, os.O_WRONLY, 0); err == nil { f.Close() return true diff --git a/src/cmd/go/internal/modload/stat_unix.go b/src/cmd/go/internal/modload/stat_unix.go index ea3b801f2c..65068444d0 100644 --- a/src/cmd/go/internal/modload/stat_unix.go +++ b/src/cmd/go/internal/modload/stat_unix.go @@ -7,6 +7,7 @@ package modload import ( + "io/fs" "os" "syscall" ) @@ -17,7 +18,7 @@ import ( // Although the root user on most Unix systems can write to files even without // permission, hasWritePerm reports false if no appropriate permission bit is // set even if the current user is root. -func hasWritePerm(path string, fi os.FileInfo) bool { +func hasWritePerm(path string, fi fs.FileInfo) bool { if os.Getuid() == 0 { // The root user can access any file, but we still want to default to // read-only mode if the go.mod file is marked as globally non-writable. diff --git a/src/cmd/go/internal/modload/stat_windows.go b/src/cmd/go/internal/modload/stat_windows.go index d7826cfc6b..0ac2391347 100644 --- a/src/cmd/go/internal/modload/stat_windows.go +++ b/src/cmd/go/internal/modload/stat_windows.go @@ -6,13 +6,11 @@ package modload -import ( - "os" -) +import "io/fs" // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(_ string, fi os.FileInfo) bool { +func hasWritePerm(_ string, fi fs.FileInfo) bool { // Windows has a read-only attribute independent of ACLs, so use that to // determine whether the file is intended to be overwritten. // diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 9f34b829fc..ab29d4d014 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -7,8 +7,8 @@ package modload import ( "errors" "fmt" + "io/fs" "io/ioutil" - "os" "path/filepath" "strings" "sync" @@ -42,7 +42,7 @@ func readVendorList() { vendorMeta = make(map[module.Version]vendorMetadata) data, err := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) } return diff --git a/src/cmd/go/internal/renameio/renameio.go b/src/cmd/go/internal/renameio/renameio.go index d573cc690d..60a7138a76 100644 --- a/src/cmd/go/internal/renameio/renameio.go +++ b/src/cmd/go/internal/renameio/renameio.go @@ -8,6 +8,7 @@ package renameio import ( "bytes" "io" + "io/fs" "math/rand" "os" "path/filepath" @@ -29,13 +30,13 @@ func Pattern(filename string) string { // final name. // // That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { +func WriteFile(filename string, data []byte, perm fs.FileMode) (err error) { return WriteToFile(filename, bytes.NewReader(data), perm) } // WriteToFile is a variant of WriteFile that accepts the data as an io.Reader // instead of a slice. -func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { +func WriteToFile(filename string, data io.Reader, perm fs.FileMode) (err error) { f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) if err != nil { return err @@ -80,7 +81,7 @@ func ReadFile(filename string) ([]byte, error) { } // tempFile creates a new temporary file with given permission bits. -func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { +func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) { for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) diff --git a/src/cmd/go/internal/renameio/umask_test.go b/src/cmd/go/internal/renameio/umask_test.go index d75d67c9a9..19e217c548 100644 --- a/src/cmd/go/internal/renameio/umask_test.go +++ b/src/cmd/go/internal/renameio/umask_test.go @@ -7,6 +7,7 @@ package renameio import ( + "io/fs" "io/ioutil" "os" "path/filepath" @@ -36,7 +37,7 @@ func TestWriteFileModeAppliesUmask(t *testing.T) { t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) } - if fi.Mode()&os.ModePerm != 0640 { - t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640) + if fi.Mode()&fs.ModePerm != 0640 { + t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&fs.ModePerm, 0640) } } diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index b1d2a9376b..e4784e9478 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -10,6 +10,7 @@ import ( "cmd/go/internal/fsys" "fmt" "go/build" + "io/fs" "os" "path" "path/filepath" @@ -128,7 +129,7 @@ func (m *Match) MatchPackages() { if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -154,7 +155,7 @@ func (m *Match) MatchPackages() { } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { + if fi.Mode()&fs.ModeSymlink != 0 && want { if target, err := os.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } @@ -264,7 +265,7 @@ func (m *Match) MatchDirs() { } } - err := fsys.Walk(dir, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(dir, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 51d333d866..00da9770df 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -12,6 +12,7 @@ import ( "fmt" "go/build" "io" + "io/fs" "io/ioutil" "os" "os/exec" @@ -1598,7 +1599,7 @@ func hashStat(name string) cache.ActionID { return h.Sum() } -func hashWriteStat(h io.Writer, info os.FileInfo) { +func hashWriteStat(h io.Writer, info fs.FileInfo) { fmt.Fprintf(h, "stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) } diff --git a/src/cmd/go/internal/vcs/vcs.go b/src/cmd/go/internal/vcs/vcs.go index 90bf10244d..7812afd488 100644 --- a/src/cmd/go/internal/vcs/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -10,6 +10,7 @@ import ( "fmt" "internal/lazyregexp" "internal/singleflight" + "io/fs" "log" urlpkg "net/url" "os" @@ -404,9 +405,9 @@ func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([ if len(args) >= 2 && args[0] == "-go-internal-mkdir" { var err error if filepath.IsAbs(args[1]) { - err = os.Mkdir(args[1], os.ModePerm) + err = os.Mkdir(args[1], fs.ModePerm) } else { - err = os.Mkdir(filepath.Join(dir, args[1]), os.ModePerm) + err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) } if err != nil { return nil, err diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index 5aa0f8e7ed..44ac24c62d 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -10,6 +10,7 @@ import ( "context" "encoding/binary" "fmt" + "io/fs" "os" "path/filepath" "runtime" @@ -87,8 +88,8 @@ func runVersion(ctx context.Context, cmd *base.Command, args []string) { // scanDir scans a directory for executables to run scanFile on. func scanDir(dir string) { - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 { scanFile(path, info, *versionV) } return nil @@ -96,7 +97,7 @@ func scanDir(dir string) { } // isExe reports whether the file should be considered executable. -func isExe(file string, info os.FileInfo) bool { +func isExe(file string, info fs.FileInfo) bool { if runtime.GOOS == "windows" { return strings.HasSuffix(strings.ToLower(file), ".exe") } @@ -107,8 +108,8 @@ func isExe(file string, info os.FileInfo) bool { // If mustPrint is true, scanFile will report any error reading file. // Otherwise (mustPrint is false, because scanFile is being called // by scanDir) scanFile prints nothing for non-Go executables. -func scanFile(file string, info os.FileInfo, mustPrint bool) { - if info.Mode()&os.ModeSymlink != 0 { +func scanFile(file string, info fs.FileInfo, mustPrint bool) { + if info.Mode()&fs.ModeSymlink != 0 { // Accept file symlinks only. i, err := os.Stat(file) if err != nil || !i.Mode().IsRegular() { diff --git a/src/cmd/go/internal/web/api.go b/src/cmd/go/internal/web/api.go index 570818843b..f7d3ed60f6 100644 --- a/src/cmd/go/internal/web/api.go +++ b/src/cmd/go/internal/web/api.go @@ -13,9 +13,9 @@ import ( "bytes" "fmt" "io" + "io/fs" "io/ioutil" "net/url" - "os" "strings" "unicode" "unicode/utf8" @@ -56,7 +56,7 @@ func (e *HTTPError) Error() string { } if err := e.Err; err != nil { - if pErr, ok := e.Err.(*os.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { + if pErr, ok := e.Err.(*fs.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { // Remove the redundant copy of the path. err = pErr.Err } @@ -67,7 +67,7 @@ func (e *HTTPError) Error() string { } func (e *HTTPError) Is(target error) bool { - return target == os.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) + return target == fs.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) } func (e *HTTPError) Unwrap() error { diff --git a/src/cmd/go/internal/web/file_test.go b/src/cmd/go/internal/web/file_test.go index 6339469045..a1bb080e07 100644 --- a/src/cmd/go/internal/web/file_test.go +++ b/src/cmd/go/internal/web/file_test.go @@ -6,6 +6,7 @@ package web import ( "errors" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -54,7 +55,7 @@ func TestGetNonexistentFile(t *testing.T) { } b, err := GetBytes(u) - if !errors.Is(err, os.ErrNotExist) { - t.Fatalf("GetBytes(%v) = %q, %v; want _, os.ErrNotExist", u, b, err) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("GetBytes(%v) = %q, %v; want _, fs.ErrNotExist", u, b, err) } } diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go index 904aee0684..e941729734 100644 --- a/src/cmd/go/internal/work/build_test.go +++ b/src/cmd/go/internal/work/build_test.go @@ -7,6 +7,7 @@ package work import ( "bytes" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -253,7 +254,7 @@ func TestRespectSetgidDir(t *testing.T) { } // Change setgiddir's permissions to include the SetGID bit. - if err := os.Chmod(setgiddir, 0755|os.ModeSetgid); err != nil { + if err := os.Chmod(setgiddir, 0755|fs.ModeSetgid); err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 824a4b5a0a..717b0cc3af 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -14,6 +14,7 @@ import ( "fmt" "internal/lazyregexp" "io" + "io/fs" "io/ioutil" "log" "math/rand" @@ -1560,7 +1561,7 @@ func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { return err } - perm := os.FileMode(0666) + perm := fs.FileMode(0666) if a1.Mode == "link" { switch cfg.BuildBuildmode { case "c-archive", "c-shared", "plugin": @@ -1609,7 +1610,7 @@ func (b *Builder) cleanup(a *Action) { } // moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN { b.Showcmd("", "mv %s %s", src, dst) return nil @@ -1635,7 +1636,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) // we have to copy the file to retain the correct permissions. // https://golang.org/issue/18878 if fi, err := os.Stat(filepath.Dir(dst)); err == nil { - if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { + if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { return b.copyFile(dst, src, perm, force) } } @@ -1670,7 +1671,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) } // copyFile is like 'cp src dst'. -func (b *Builder) copyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) copyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN || cfg.BuildX { b.Showcmd("", "cp %s %s", src, dst) if cfg.BuildN { -- cgit v1.2.1 From 310984bf54a52b15085e195a402873ab558d34d4 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 19 Oct 2020 12:01:23 +0200 Subject: syscall, cmd/go/internal/modload: add and use Access on aix Implement Access using Faccessat on aix following golang.org/x/sys/unix CL 262897 and switch cmd/go/internal/modload to use it to implement hasWritePerm. Change-Id: I682e44737ac2bac5a203ac1c9ddd277810454426 Reviewed-on: https://go-review.googlesource.com/c/go/+/263540 Trust: Tobias Klauser Run-TryBot: Tobias Klauser TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modload/stat_openfile.go | 2 +- src/cmd/go/internal/modload/stat_unix.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/stat_openfile.go b/src/cmd/go/internal/modload/stat_openfile.go index 7cdeaf47a2..5842b858f0 100644 --- a/src/cmd/go/internal/modload/stat_openfile.go +++ b/src/cmd/go/internal/modload/stat_openfile.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix js,wasm plan9 +// +build js,wasm plan9 // On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions // are checked by the server and group information is not known to the client, diff --git a/src/cmd/go/internal/modload/stat_unix.go b/src/cmd/go/internal/modload/stat_unix.go index 65068444d0..f49278ec3a 100644 --- a/src/cmd/go/internal/modload/stat_unix.go +++ b/src/cmd/go/internal/modload/stat_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package modload -- cgit v1.2.1 From 0bf507efe9f995076e3a65bcf61baf3e905b58c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 19 Oct 2020 15:06:02 +0100 Subject: cmd/go: add BuildID to list -json -export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That is, the following two pieces of shell code are now equivalent: $ go tool buildid $(go list -export -f {{.Export}} strings) v_0VqA6yzwuMg2dn4u57/PXcIR2Pb8Mi9yRdcdkwe $ go list -export -f {{.BuildID}} strings v_0VqA6yzwuMg2dn4u57/PXcIR2Pb8Mi9yRdcdkwe This does not expose any information that wasn't available before, but makes this workflow simpler and faster. In the first example, we have to execute two programs, and 'go tool buildid' has to re-open the export data file to read the build ID. With the new mechanism, 'go list -export' already has the build ID ready, so we can simply print it out. Moreover, when listing lots of related packages like './...', we can now obtain all their build IDs at once. Fixes #37281. Change-Id: I8e2f65a08391b3df1a628c6e06e708b8c8cb7865 Reviewed-on: https://go-review.googlesource.com/c/go/+/263542 Trust: Daniel Martí Trust: Bryan C. Mills Run-TryBot: Daniel Martí TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/list/list.go | 1 + src/cmd/go/internal/load/pkg.go | 1 + src/cmd/go/internal/work/buildid.go | 1 + src/cmd/go/internal/work/exec.go | 1 + 4 files changed, 4 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 732cebc8cb..9fd9d7446d 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -66,6 +66,7 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is: BinaryOnly bool // binary-only package (no longer supported) ForTest string // package is only for use in named test Export string // file containing export data (when using -export) + BuildID string // build ID of the export data (when using -export) Module *Module // info about package's containing module, if any (can be nil) Match []string // command-line patterns matching this package DepOnly bool // package is only a dependency, not explicitly listed diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 066ff6c981..913b3b94d7 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -60,6 +60,7 @@ type PackagePublic struct { ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory ForTest string `json:",omitempty"` // package is only for use in named test Export string `json:",omitempty"` // file containing export data (set by go list -export) + BuildID string `json:",omitempty"` // build ID of the export data (set by go list -export) Module *modinfo.ModulePublic `json:",omitempty"` // info about package's module, if any Match []string `json:",omitempty"` // command-line patterns matching this package Goroot bool `json:",omitempty"` // is this package found in the Go root? diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index a3c9b1a2c1..5cd3124e54 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -713,6 +713,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { return err } a.Package.Export = c.OutputFile(outputID) + a.Package.BuildID = a.buildID } } } diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 717b0cc3af..3ffdca5718 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -433,6 +433,7 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { need &^= needBuild if b.NeedExport { p.Export = a.built + p.BuildID = a.buildID } if need&needCompiledGoFiles != 0 { if err := b.loadCachedSrcFiles(a); err == nil { -- cgit v1.2.1 From f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 19 Oct 2020 20:45:01 -0400 Subject: cmd/go/internal/renameio: include ios in the darwin test-flake mitigation Because the "ios" build constraint implies "darwin", it is already included in the general "darwin" flakiness workaround in cmd/go/internal/robustio. We just need to relax the renameio test to avoid false-positives there. I do not expect this change to drive the rate of false-positives down to zero, but it should at least reduce noise on the build dashboard. For #42066 Change-Id: Ia33dbd33295fce5b3261b4831f2807ce29b82e65 Reviewed-on: https://go-review.googlesource.com/c/go/+/263777 Trust: Bryan C. Mills Reviewed-by: Cherry Zhang Reviewed-by: Dmitri Shuralyov --- src/cmd/go/internal/renameio/renameio_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/renameio/renameio_test.go b/src/cmd/go/internal/renameio/renameio_test.go index df8ddabdb8..e6d2025a0e 100644 --- a/src/cmd/go/internal/renameio/renameio_test.go +++ b/src/cmd/go/internal/renameio/renameio_test.go @@ -144,10 +144,12 @@ func TestConcurrentReadsAndWrites(t *testing.T) { // As long as those are the only errors and *some* of the reads succeed, we're happy. minReadSuccesses = attempts / 4 - case "darwin": - // The filesystem on macOS 10.14 occasionally fails with "no such file or - // directory" errors. See https://golang.org/issue/33041. The flake rate is - // fairly low, so ensure that at least 75% of attempts succeed. + case "darwin", "ios": + // The filesystem on certain versions of macOS (10.14) and iOS (affected + // versions TBD) occasionally fail with "no such file or directory" errors. + // See https://golang.org/issue/33041 and https://golang.org/issue/42066. + // The flake rate is fairly low, so ensure that at least 75% of attempts + // succeed. minReadSuccesses = attempts - (attempts / 4) } -- cgit v1.2.1 From d595712540f00d980b1276ed25495ee7e05c1bfa Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 20 Oct 2020 09:06:42 -0400 Subject: cmd/asm: rename "compiling runtime" flag Rename the assembler "-compilingRuntime" flag to "-compiling-runtime", to be more consistent with the flag style of other Go commands. Change-Id: I8cc5cbf0b9b34d1dd4e9fa499d3fec8c1ef10b6e Reviewed-on: https://go-review.googlesource.com/c/go/+/263857 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/go/internal/work/gc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 2df4a52ba5..e93031431c 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -293,7 +293,7 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } if objabi.IsRuntimePackagePath(pkgpath) { - args = append(args, "-compilingRuntime") + args = append(args, "-compiling-runtime") if objabi.Regabi_enabled != 0 { // In order to make it easier to port runtime assembly // to the register ABI, we introduce a macro -- cgit v1.2.1 From 1b09d430678d4a6f73b2443463d11f75851aba8a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 16 Oct 2020 00:49:02 -0400 Subject: all: update references to symbols moved from io/ioutil to io The old ioutil references are still valid, but update our code to reflect best practices and get used to the new locations. Code compiled with the bootstrap toolchain (cmd/asm, cmd/dist, cmd/compile, debug/elf) must remain Go 1.4-compatible and is excluded. Also excluded vendored code. For #41190. Change-Id: I6d86f2bf7bc37a9d904b6cee3fe0c7af6d94d5b1 Reviewed-on: https://go-review.googlesource.com/c/go/+/263142 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Emmanuel Odeke --- src/cmd/go/internal/clean/clean.go | 3 ++- src/cmd/go/internal/fsys/fsys_test.go | 3 ++- src/cmd/go/internal/imports/read.go | 4 ++-- src/cmd/go/internal/lockedfile/lockedfile.go | 5 ++--- src/cmd/go/internal/modfetch/codehost/git.go | 5 ++--- src/cmd/go/internal/modfetch/codehost/git_test.go | 3 ++- src/cmd/go/internal/modfetch/codehost/shell.go | 3 ++- src/cmd/go/internal/modfetch/coderepo.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 2 +- src/cmd/go/internal/modfetch/proxy.go | 3 +-- src/cmd/go/internal/modfetch/sumdb.go | 4 ++-- src/cmd/go/internal/web/api.go | 5 ++--- 12 files changed, 21 insertions(+), 21 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 6bfd7ae21e..095c3cc713 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -8,6 +8,7 @@ package clean import ( "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -172,7 +173,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { f, err := lockedfile.Edit(filepath.Join(dir, "testexpire.txt")) if err == nil { now := time.Now().UnixNano() - buf, _ := ioutil.ReadAll(f) + buf, _ := io.ReadAll(f) prev, _ := strconv.ParseInt(strings.TrimSpace(string(buf)), 10, 64) if now > prev { if err = f.Truncate(0); err == nil { diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index ba9f05d00b..28c3f08cb9 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "internal/testenv" + "io" "io/fs" "io/ioutil" "os" @@ -418,7 +419,7 @@ this can exist because the parent directory is deleted t.Errorf("Open(%q): got error %v, want nil", tc.path, err) continue } - contents, err := ioutil.ReadAll(f) + contents, err := io.ReadAll(f) if err != nil { t.Errorf("unexpected error reading contents of file: %v", err) } diff --git a/src/cmd/go/internal/imports/read.go b/src/cmd/go/internal/imports/read.go index 58c2abdc29..5e270781d7 100644 --- a/src/cmd/go/internal/imports/read.go +++ b/src/cmd/go/internal/imports/read.go @@ -198,7 +198,7 @@ func (r *importReader) readImport(imports *[]string) { r.readString(imports) } -// ReadComments is like ioutil.ReadAll, except that it only reads the leading +// ReadComments is like io.ReadAll, except that it only reads the leading // block of comments in the file. func ReadComments(f io.Reader) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} @@ -210,7 +210,7 @@ func ReadComments(f io.Reader) ([]byte, error) { return r.buf, r.err } -// ReadImports is like ioutil.ReadAll, except that it expects a Go file as input +// ReadImports is like io.ReadAll, except that it expects a Go file as input // and stops reading the input once the imports have completed. func ReadImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} diff --git a/src/cmd/go/internal/lockedfile/lockedfile.go b/src/cmd/go/internal/lockedfile/lockedfile.go index 503024da4b..82e1a89675 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile.go +++ b/src/cmd/go/internal/lockedfile/lockedfile.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "os" "runtime" ) @@ -104,7 +103,7 @@ func Read(name string) ([]byte, error) { } defer f.Close() - return ioutil.ReadAll(f) + return io.ReadAll(f) } // Write opens the named file (creating it with the given permissions if needed), @@ -136,7 +135,7 @@ func Transform(name string, t func([]byte) ([]byte, error)) (err error) { } defer f.Close() - old, err := ioutil.ReadAll(f) + old, err := io.ReadAll(f) if err != nil { return err } diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go index 58b4b2f2d3..8abc039e7f 100644 --- a/src/cmd/go/internal/modfetch/codehost/git.go +++ b/src/cmd/go/internal/modfetch/codehost/git.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "net/url" "os" "os/exec" @@ -832,7 +831,7 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, return nil, err } - return ioutil.NopCloser(bytes.NewReader(archive)), nil + return io.NopCloser(bytes.NewReader(archive)), nil } // ensureGitAttributes makes sure export-subst and export-ignore features are @@ -863,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) { } }() - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return err } diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go index 16908b3e84..981e3fe91f 100644 --- a/src/cmd/go/internal/modfetch/codehost/git_test.go +++ b/src/cmd/go/internal/modfetch/codehost/git_test.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "internal/testenv" + "io" "io/fs" "io/ioutil" "log" @@ -433,7 +434,7 @@ func TestReadZip(t *testing.T) { if tt.err != "" { t.Fatalf("ReadZip: no error, wanted %v", tt.err) } - zipdata, err := ioutil.ReadAll(rc) + zipdata, err := io.ReadAll(rc) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/codehost/shell.go b/src/cmd/go/internal/modfetch/codehost/shell.go index 2762c55720..b9525adf5e 100644 --- a/src/cmd/go/internal/modfetch/codehost/shell.go +++ b/src/cmd/go/internal/modfetch/codehost/shell.go @@ -14,6 +14,7 @@ import ( "bytes" "flag" "fmt" + "io" "io/ioutil" "log" "os" @@ -115,7 +116,7 @@ func main() { fmt.Fprintf(os.Stderr, "?%s\n", err) continue } - data, err := ioutil.ReadAll(rc) + data, err := io.ReadAll(rc) rc.Close() if err != nil { fmt.Fprintf(os.Stderr, "?%s\n", err) diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go index 7f44e18a70..b6bcf83f1a 100644 --- a/src/cmd/go/internal/modfetch/coderepo.go +++ b/src/cmd/go/internal/modfetch/coderepo.go @@ -1052,7 +1052,7 @@ type dataFile struct { func (f dataFile) Path() string { return f.name } func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil } func (f dataFile) Open() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(f.data)), nil + return io.NopCloser(bytes.NewReader(f.data)), nil } type dataFileInfo struct { diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 6ff455e89c..40196c4e9a 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -461,7 +461,7 @@ func checkMod(mod module.Version) { // goModSum returns the checksum for the go.mod contents. func goModSum(data []byte) (string, error) { return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil }) } diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 819990b403..d75b4da521 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "net/url" "path" pathpkg "path" @@ -305,7 +304,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) { return nil, err } defer body.Close() - return ioutil.ReadAll(body) + return io.ReadAll(body) } func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 5108961a33..4fbc54d15c 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -12,8 +12,8 @@ import ( "bytes" "errors" "fmt" + "io" "io/fs" - "io/ioutil" "net/url" "os" "path/filepath" @@ -228,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error { return err } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } diff --git a/src/cmd/go/internal/web/api.go b/src/cmd/go/internal/web/api.go index f7d3ed60f6..9053b16b62 100644 --- a/src/cmd/go/internal/web/api.go +++ b/src/cmd/go/internal/web/api.go @@ -14,7 +14,6 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "net/url" "strings" "unicode" @@ -87,7 +86,7 @@ func GetBytes(u *url.URL) ([]byte, error) { if err := resp.Err(); err != nil { return nil, err } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading %s: %v", u.Redacted(), err) } @@ -130,7 +129,7 @@ func (r *Response) formatErrorDetail() string { } // Ensure that r.errorDetail has been populated. - _, _ = io.Copy(ioutil.Discard, r.Body) + _, _ = io.Copy(io.Discard, r.Body) s := r.errorDetail.buf.String() if !utf8.ValidString(s) { -- cgit v1.2.1 From e2c420591cbbd684594a111fa5cdeb40c68964a5 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 19 Oct 2020 14:02:14 -0400 Subject: cmd/go/internal/modload: remove printStackInDie functionality Previously, when running cmd/go tests, if the module root directory is requested when modules are explicitly disabled, we printed a stack trace in addition to the error message that's normally printed. The stack trace isn't that useful, and it makes the actual error hard to find. Change-Id: I8230d668f3f16659f08d0d685124c41b4055c5b9 Reviewed-on: https://go-review.googlesource.com/c/go/+/263659 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modload/build.go | 4 ---- src/cmd/go/internal/modload/init.go | 10 ---------- src/cmd/go/internal/modload/testgo.go | 11 ----------- 3 files changed, 25 deletions(-) delete mode 100644 src/cmd/go/internal/modload/testgo.go (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index f49b52df56..b9abb0b93c 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -13,7 +13,6 @@ import ( "internal/goroot" "os" "path/filepath" - "runtime/debug" "strings" "cmd/go/internal/base" @@ -312,9 +311,6 @@ func mustFindModule(target, path string) module.Version { return Target } - if printStackInDie { - debug.PrintStack() - } base.Fatalf("build %v: cannot find module for path %v", target, path) panic("unreachable") } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index e1b784860b..1fcc53735c 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -16,7 +16,6 @@ import ( "os" "path" "path/filepath" - "runtime/debug" "strconv" "strings" @@ -330,16 +329,7 @@ func ModFilePath() string { return filepath.Join(modRoot, "go.mod") } -// printStackInDie causes die to print a stack trace. -// -// It is enabled by the testgo tag, and helps to diagnose paths that -// unexpectedly require a main module. -var printStackInDie = false - func die() { - if printStackInDie { - debug.PrintStack() - } if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } diff --git a/src/cmd/go/internal/modload/testgo.go b/src/cmd/go/internal/modload/testgo.go deleted file mode 100644 index 6b34f5be39..0000000000 --- a/src/cmd/go/internal/modload/testgo.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build testgo - -package modload - -func init() { - printStackInDie = true -} -- cgit v1.2.1 From 5e9582e3f0d10523d32a25a338cbade21266dca3 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 15 Oct 2020 11:45:32 -0400 Subject: cmd/go: support overlays for synthesized packages. The main missing piece here was supporting Stat in the overlay filesystem, in the parts of the package code that determines whether an command line argument is a file on disk or a directory. so this change adds a Stat function to the fsys package. It's implemented the same way as the already existing fsys.lstat function, but instead of os.Lstat, it calls os.Stat on disk files. Then, the change changes parts of the package code to use the overlay Stat instead of the os package's Stat. For #39958 Change-Id: I8e478ae386f05b48d7dd71bd7e47584f090623df Reviewed-on: https://go-review.googlesource.com/c/go/+/262617 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/fsys/fsys.go | 20 +++-- src/cmd/go/internal/fsys/fsys_test.go | 155 +++++++++++++++++++++++++++++++++- src/cmd/go/internal/load/pkg.go | 7 +- 3 files changed, 170 insertions(+), 12 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 67359ffb6d..814e323701 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -434,29 +434,39 @@ func Walk(root string, walkFn filepath.WalkFunc) error { // lstat implements a version of os.Lstat that operates on the overlay filesystem. func lstat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Lstat, "lstat") +} + +// Stat implements a version of os.Stat that operates on the overlay filesystem. +func Stat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Stat, "stat") +} + +// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). +func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { cpath := canonicalize(path) if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { - return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} + return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} } node, ok := overlay[cpath] if !ok { // The file or directory is not overlaid. - return os.Lstat(cpath) + return osStat(path) } switch { case node.isDeleted(): return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} case node.isDir(): - return fakeDir(filepath.Base(cpath)), nil + return fakeDir(filepath.Base(path)), nil default: - fi, err := os.Lstat(node.actualFilePath) + fi, err := osStat(node.actualFilePath) if err != nil { return nil, err } - return fakeFile{name: filepath.Base(cpath), real: fi}, nil + return fakeFile{name: filepath.Base(path), real: fi}, nil } } diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 28c3f08cb9..19bf282190 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -506,7 +506,7 @@ func TestWalk(t *testing.T) { `, ".", []file{ - {".", "root", 0, fs.ModeDir | 0700, true}, + {".", ".", 0, fs.ModeDir | 0700, true}, {"file.txt", "file.txt", 0, 0600, false}, }, }, @@ -522,7 +522,7 @@ contents of other file `, ".", []file{ - {".", "root", 0, fs.ModeDir | 0500, true}, + {".", ".", 0, fs.ModeDir | 0500, true}, {"file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, }, @@ -538,7 +538,7 @@ contents of other file `, ".", []file{ - {".", "root", 0, fs.ModeDir | 0500, true}, + {".", ".", 0, fs.ModeDir | 0500, true}, {"file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, }, @@ -554,7 +554,7 @@ contents of other file `, ".", []file{ - {".", "root", 0, fs.ModeDir | 0500, true}, + {".", ".", 0, fs.ModeDir | 0500, true}, {"dir", "dir", 0, fs.ModeDir | 0500, true}, {"dir" + string(filepath.Separator) + "file.txt", "file.txt", 23, 0600, false}, {"other.txt", "other.txt", 23, 0600, false}, @@ -818,3 +818,150 @@ contents`, }) } } + +func TestStat(t *testing.T) { + testenv.MustHaveSymlink(t) + + type file struct { + name string + size int64 + mode os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | os.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | os.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := Stat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("Stat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("Stat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("Stat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(os.ModeDir|0700) != tc.want.mode { + t.Errorf("Stat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("Stat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("Stat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} + +func TestStat_Symlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + initOverlay(t, `{ + "Replace": {"file.go": "symlink"} +} +-- to.go -- +0123456789 +`) + + // Create symlink + if err := os.Symlink("to.go", "symlink"); err != nil { + t.Error(err) + } + + f := "file.go" + fi, err := Stat(f) + if err != nil { + t.Errorf("Stat(%q): got error %q, want nil error", f, err) + } + + if !fi.Mode().IsRegular() { + t.Errorf("Stat(%q).Mode(): got %v, want regular mode", f, fi.Mode()) + } + + if fi.Size() != 11 { + t.Errorf("Stat(%q).Size(): got %v, want 11", f, fi.Size()) + } +} diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 913b3b94d7..2bdc08ba36 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -28,6 +28,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/par" @@ -977,7 +978,7 @@ var isDirCache par.Cache func isDir(path string) bool { return isDirCache.Do(path, func() interface{} { - fi, err := os.Stat(path) + fi, err := fsys.Stat(path) return err == nil && fi.IsDir() }).(bool) } @@ -2145,7 +2146,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if strings.HasSuffix(p, ".go") { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). - if fi, err := os.Stat(p); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(p); err == nil && !fi.IsDir() { return []*Package{GoFilesPackage(ctx, patterns)} } } @@ -2305,7 +2306,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { var dirent []fs.FileInfo var dir string for _, file := range gofiles { - fi, err := os.Stat(file) + fi, err := fsys.Stat(file) if err != nil { base.Fatalf("%s", err) } -- cgit v1.2.1 From 15ead857dbc638b9d83a7686acf0dc746fc45918 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Wed, 9 Sep 2020 17:24:23 -0500 Subject: cmd/compiler,cmd/go,sync: add internal {LoadAcq,StoreRel}64 on ppc64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an internal atomic intrinsic for load with acquire semantics (extending LoadAcq to 64b) and add LoadAcquintptr for internal use within the sync package. For other arches, this remaps to the appropriate atomic.Load{,64} intrinsic which should not alter code generation. Similarly, add StoreRel{uintptr,64} for consistency, and inline. Finally, add an exception to allow sync to directly use the runtime/internal/atomic package which avoids more convoluted workarounds (contributed by Lynn Boger). In an extreme example, sync.(*Pool).pin consumes 20% of wall time during fmt tests. This is reduced to 5% on ppc64le/power9. From the fmt benchmarks on ppc64le: name old time/op new time/op delta SprintfPadding 468ns ± 0% 451ns ± 0% -3.63% SprintfEmpty 73.3ns ± 0% 51.9ns ± 0% -29.20% SprintfString 135ns ± 0% 122ns ± 0% -9.63% SprintfTruncateString 232ns ± 0% 214ns ± 0% -7.76% SprintfTruncateBytes 216ns ± 0% 202ns ± 0% -6.48% SprintfSlowParsingPath 162ns ± 0% 142ns ± 0% -12.35% SprintfQuoteString 1.00µs ± 0% 0.99µs ± 0% -1.39% SprintfInt 117ns ± 0% 104ns ± 0% -11.11% SprintfIntInt 190ns ± 0% 175ns ± 0% -7.89% SprintfPrefixedInt 232ns ± 0% 212ns ± 0% -8.62% SprintfFloat 270ns ± 0% 255ns ± 0% -5.56% SprintfComplex 1.01µs ± 0% 0.99µs ± 0% -1.68% SprintfBoolean 127ns ± 0% 111ns ± 0% -12.60% SprintfHexString 220ns ± 0% 198ns ± 0% -10.00% SprintfHexBytes 261ns ± 0% 252ns ± 0% -3.45% SprintfBytes 600ns ± 0% 590ns ± 0% -1.67% SprintfStringer 684ns ± 0% 658ns ± 0% -3.80% SprintfStructure 2.57µs ± 0% 2.57µs ± 0% -0.12% ManyArgs 669ns ± 0% 646ns ± 0% -3.44% FprintInt 140ns ± 0% 136ns ± 0% -2.86% FprintfBytes 184ns ± 0% 181ns ± 0% -1.63% FprintIntNoAlloc 140ns ± 0% 136ns ± 0% -2.86% ScanInts 929µs ± 0% 921µs ± 0% -0.79% ScanRecursiveInt 122ms ± 0% 121ms ± 0% -0.11% ScanRecursiveIntReaderWrapper 122ms ± 0% 122ms ± 0% -0.18% Change-Id: I4d66780261b57b06ef600229e475462e7313f0d6 Reviewed-on: https://go-review.googlesource.com/c/go/+/253748 Run-TryBot: Lynn Boger Reviewed-by: Lynn Boger Reviewed-by: Keith Randall Trust: Lynn Boger TryBot-Result: Go Bot --- src/cmd/go/internal/load/pkg.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 2bdc08ba36..c9665265e9 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1337,6 +1337,11 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * return p } + // Allow sync package to access lightweight atomic functions limited to the runtime. + if p.Standard && strings.HasPrefix(importerPath, "sync") && p.ImportPath == "runtime/internal/atomic" { + return p + } + // Internal is present. // Map import path back to directory corresponding to parent of internal. if i > 0 { -- cgit v1.2.1 From bcc333348769efed7c38acfa013e5475c53e8f5f Mon Sep 17 00:00:00 2001 From: Obeyda Djeffal Date: Fri, 16 Oct 2020 16:34:15 +0100 Subject: cmd/go: ignore GOFLAGS values without name in go env/bug This happens with 'go env' and 'go bug'. If GOFLAGS variable is set to something like '=value', running `go env` panics with this error message: goroutine 1 [running]: cmd/go/internal/base.SetFromGOFLAGS(0xd96838) cmd/go/internal/base/goflags.go:101 +0x9a7 main.main() cmd/go/main.go:188 +0x755 This happens when the 'name' of the flag is not specified ('=' or '=value'), with any combination of other flags. Other commands show this error message: go: parsing $GOFLAGS: non-flag This happens only with 'env' and 'bug' because we have this: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/go/internal/base/goflags.go#40 New behaviour: ignore the bad flag, since we don't want to report that with `go env` or `go bug`. Fixes: #42013 Change-Id: I72602840ca00293d2a92ea28451b75b9799e3d6c Reviewed-on: https://go-review.googlesource.com/c/go/+/263098 Reviewed-by: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Trust: Michael Matloob --- src/cmd/go/internal/base/goflags.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/base/goflags.go b/src/cmd/go/internal/base/goflags.go index 4da27550fd..267006be7a 100644 --- a/src/cmd/go/internal/base/goflags.go +++ b/src/cmd/go/internal/base/goflags.go @@ -92,7 +92,11 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } for _, goflag := range goflags { name, value, hasValue := goflag, "", false - if i := strings.Index(goflag, "="); i >= 0 { + // Ignore invalid flags like '=' or '=value'. + // If it is not reported in InitGOFlags it means we don't want to report it. + if i := strings.Index(goflag, "="); i == 0 { + continue + } else if i > 0 { name, value, hasValue = goflag[:i], goflag[i+1:], true } if strings.HasPrefix(name, "--") { -- cgit v1.2.1 From 214136b7412d56151d5443741feb0ed873facf2e Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 21 Oct 2020 10:07:02 -0400 Subject: cmd/go/internal/fsys: use a root other than "." in Walk tests Fixes #42115 Change-Id: Icf4c9eac5ed3295acbc8377c7a06f82c6bddc747 Reviewed-on: https://go-review.googlesource.com/c/go/+/264177 Trust: Bryan C. Mills Trust: Jay Conrod Run-TryBot: Bryan C. Mills Reviewed-by: Jay Conrod Reviewed-by: David du Colombier <0intro@gmail.com> TryBot-Result: Go Bot --- src/cmd/go/internal/fsys/fsys_test.go | 74 +++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 34 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 19bf282190..fd98d13f3d 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -487,6 +487,11 @@ contents don't matter for this test } func TestWalk(t *testing.T) { + // The root of the walk must be a name with an actual basename, not just ".". + // Walk uses Lstat to obtain the name of the root, and Lstat on platforms + // other than Plan 9 reports the name "." instead of the actual base name of + // the directory. (See https://golang.org/issue/42115.) + type file struct { path string name string @@ -502,62 +507,62 @@ func TestWalk(t *testing.T) { }{ {"no overlay", ` {} --- file.txt -- +-- dir/file.txt -- `, - ".", + "dir", []file{ - {".", ".", 0, fs.ModeDir | 0700, true}, - {"file.txt", "file.txt", 0, 0600, false}, + {"dir", "dir", 0, fs.ModeDir | 0700, true}, + {"dir/file.txt", "file.txt", 0, 0600, false}, }, }, {"overlay with different file", ` { "Replace": { - "file.txt": "other.txt" + "dir/file.txt": "dir/other.txt" } } --- file.txt -- --- other.txt -- +-- dir/file.txt -- +-- dir/other.txt -- contents of other file `, - ".", + "dir", []file{ - {".", ".", 0, fs.ModeDir | 0500, true}, - {"file.txt", "file.txt", 23, 0600, false}, - {"other.txt", "other.txt", 23, 0600, false}, + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, }, }, {"overlay with new file", ` { "Replace": { - "file.txt": "other.txt" + "dir/file.txt": "dir/other.txt" } } --- other.txt -- +-- dir/other.txt -- contents of other file `, - ".", + "dir", []file{ - {".", ".", 0, fs.ModeDir | 0500, true}, - {"file.txt", "file.txt", 23, 0600, false}, - {"other.txt", "other.txt", 23, 0600, false}, + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, }, }, {"overlay with new directory", ` { "Replace": { - "dir/file.txt": "other.txt" + "dir/subdir/file.txt": "dir/other.txt" } } --- other.txt -- +-- dir/other.txt -- contents of other file `, - ".", + "dir", []file{ - {".", ".", 0, fs.ModeDir | 0500, true}, {"dir", "dir", 0, fs.ModeDir | 0500, true}, - {"dir" + string(filepath.Separator) + "file.txt", "file.txt", 23, 0600, false}, - {"other.txt", "other.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + {"dir/subdir", "subdir", 0, fs.ModeDir | 0500, true}, + {"dir/subdir/file.txt", "file.txt", 23, 0600, false}, }, }, } @@ -576,8 +581,9 @@ contents of other file t.Errorf("Walk: saw %#v in walk; want %#v", got, tc.wantFiles) } for i := 0; i < len(got) && i < len(tc.wantFiles); i++ { - if got[i].path != tc.wantFiles[i].path { - t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, tc.wantFiles[i].path) + wantPath := filepath.FromSlash(tc.wantFiles[i].path) + if got[i].path != wantPath { + t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, wantPath) } if got[i].name != tc.wantFiles[i].name { t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name) @@ -603,24 +609,24 @@ func TestWalk_SkipDir(t *testing.T) { initOverlay(t, ` { "Replace": { - "skipthisdir/file.go": "dummy.txt", - "dontskip/file.go": "dummy.txt", - "dontskip/skip/file.go": "dummy.txt" + "dir/skip/file.go": "dummy.txt", + "dir/dontskip/file.go": "dummy.txt", + "dir/dontskip/skip/file.go": "dummy.txt" } } -- dummy.txt -- `) var seen []string - Walk(".", func(path string, info fs.FileInfo, err error) error { - seen = append(seen, path) - if path == "skipthisdir" || path == filepath.Join("dontskip", "skip") { + Walk("dir", func(path string, info fs.FileInfo, err error) error { + seen = append(seen, filepath.ToSlash(path)) + if info.Name() == "skip" { return filepath.SkipDir } return nil }) - wantSeen := []string{".", "dontskip", filepath.Join("dontskip", "file.go"), filepath.Join("dontskip", "skip"), "dummy.txt", "skipthisdir"} + wantSeen := []string{"dir", "dir/dontskip", "dir/dontskip/file.go", "dir/dontskip/skip", "dir/skip"} if len(seen) != len(wantSeen) { t.Errorf("paths seen in walk: got %v entries; want %v entries", len(seen), len(wantSeen)) @@ -675,8 +681,8 @@ func TestWalk_Symlink(t *testing.T) { wantFiles []string }{ {"control", "dir", []string{"dir", "dir" + string(filepath.Separator) + "file"}}, - // ensure Walk doesn't wolk into the directory pointed to by the symlink - // (because it's supposed to use Lstat instead of Stat. + // ensure Walk doesn't walk into the directory pointed to by the symlink + // (because it's supposed to use Lstat instead of Stat). {"symlink_to_dir", "symlink", []string{"symlink"}}, {"overlay_to_symlink_to_dir", "overlay_symlink", []string{"overlay_symlink"}}, } -- cgit v1.2.1 From 9c28a50fd18b2e69c43dec9b871e96bab6ec4ede Mon Sep 17 00:00:00 2001 From: hitzhangjie Date: Wed, 21 Oct 2020 05:09:26 +0000 Subject: cmd/go: when module enabled, `go clean` removes built binary Now "go clean" can remove binary as expected, when module enabled and the module name isn't "main" or the name of folder. Fixes issue #41656 Change-Id: I54b9435ece045e03a12dc230efe84c8dd381a07c GitHub-Last-Rev: f4ea2d8c765a6e3a2b89fe2c9fb5b2c80551efa5 GitHub-Pull-Request: golang/go#41999 Reviewed-on: https://go-review.googlesource.com/c/go/+/262677 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills Trust: Jay Conrod Trust: Bryan C. Mills --- src/cmd/go/internal/clean/clean.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 095c3cc713..87933f04f3 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -276,6 +276,8 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem, elem+".exe", + p.DefaultExecName(), + p.DefaultExecName()+".exe", ) } @@ -283,16 +285,28 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem+".test", elem+".test.exe", + p.DefaultExecName()+".test", + p.DefaultExecName()+".test.exe", ) - // Remove a potential executable for each .go file in the directory that + // Remove a potential executable, test executable for each .go file in the directory that // is not part of the directory's package. for _, dir := range dirs { name := dir.Name() if packageFile[name] { continue } - if !dir.IsDir() && strings.HasSuffix(name, ".go") { + + if dir.IsDir() { + continue + } + + if strings.HasSuffix(name, "_test.go") { + base := name[:len(name)-len("_test.go")] + allRemove = append(allRemove, base+".test", base+".test.exe") + } + + if strings.HasSuffix(name, ".go") { // TODO(adg,rsc): check that this .go file is actually // in "package main", and therefore capable of building // to an executable file. -- cgit v1.2.1 From ed2010e676042b387d72daaa4c39b54c4c57c031 Mon Sep 17 00:00:00 2001 From: Keyuan Date: Tue, 20 Oct 2020 19:47:29 -0700 Subject: imports: make ScanDir ignore go files start with dot Adding "." Prefix Check for go files. Fixes #42047 Change-Id: Ifc42bf562f52fdd304f9828b06fc57888fcd8049 Reviewed-on: https://go-review.googlesource.com/c/go/+/264078 Reviewed-by: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Trust: Jay Conrod Trust: Bryan C. Mills --- src/cmd/go/internal/imports/scan.go | 2 +- src/cmd/go/internal/imports/testdata/android/.h.go | 3 +++ src/cmd/go/internal/imports/testdata/illumos/.h.go | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 src/cmd/go/internal/imports/testdata/android/.h.go create mode 100644 src/cmd/go/internal/imports/testdata/illumos/.h.go (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/imports/scan.go b/src/cmd/go/internal/imports/scan.go index d45393f36c..d7e674b129 100644 --- a/src/cmd/go/internal/imports/scan.go +++ b/src/cmd/go/internal/imports/scan.go @@ -34,7 +34,7 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { } } - if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { + if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { files = append(files, filepath.Join(dir, name)) } } diff --git a/src/cmd/go/internal/imports/testdata/android/.h.go b/src/cmd/go/internal/imports/testdata/android/.h.go new file mode 100644 index 0000000000..53c529e777 --- /dev/null +++ b/src/cmd/go/internal/imports/testdata/android/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" diff --git a/src/cmd/go/internal/imports/testdata/illumos/.h.go b/src/cmd/go/internal/imports/testdata/illumos/.h.go new file mode 100644 index 0000000000..53c529e777 --- /dev/null +++ b/src/cmd/go/internal/imports/testdata/illumos/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" -- cgit v1.2.1 From 61313dab524f2c82add8442e15d87fca5b7103de Mon Sep 17 00:00:00 2001 From: Xiangdong Ji Date: Sun, 18 Oct 2020 11:43:23 -0700 Subject: cmd/go: use the last -linkmode flag to determine external linking Current linkmode checking in determining package dependencies doesn't take multiple -linkmode options into consideration, may lead to redundant dependency on 'runtime/cgo'. Fixes the problem and adds a testcase. Change-Id: Iac5ea9fb3ca5ef931201afd0f3441f41f946c919 Reviewed-on: https://go-review.googlesource.com/c/go/+/263497 Trust: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/load/pkg.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index c9665265e9..f07bd3e075 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1978,16 +1978,20 @@ func externalLinkingForced(p *Package) bool { // external linking mode, as of course does // -ldflags=-linkmode=external. External linking mode forces // an import of runtime/cgo. + // If there are multiple -linkmode options, the last one wins. pieCgo := cfg.BuildBuildmode == "pie" && !sys.InternalLinkPIESupported(cfg.BuildContext.GOOS, cfg.BuildContext.GOARCH) linkmodeExternal := false if p != nil { ldflags := BuildLdflags.For(p) - for i, a := range ldflags { - if a == "-linkmode=external" { - linkmodeExternal = true - } - if a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { + for i := len(ldflags) - 1; i >= 0; i-- { + a := ldflags[i] + if a == "-linkmode=external" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { linkmodeExternal = true + break + } else if a == "-linkmode=internal" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "internal" { + break } } } -- cgit v1.2.1 From de74ea5d740ccc69dbb146578dc8a965351a3d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 17 Oct 2020 20:03:19 +0100 Subject: cmd/go: set TOOLEXEC_IMPORTPATH for -toolexec tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way, a -toolexec tool can tell precisely what package is being built when it's run. This was very hard to do before, because the tool had to piece together that information given the build action's arguments or flags. Since there wasn't a good set of tests for -toolexec, add one in the form of a test script. It builds a simple set of packages with a variety of build tools, to ensure that all the cases behave as expected. Like other recent master changes, include the changelog item for this user-facing change too. Fixes #15677. Change-Id: I0a5a1d9485840323ec138b2e64b7e7dd803fdf90 Reviewed-on: https://go-review.googlesource.com/c/go/+/263357 Run-TryBot: Daniel Martí TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Trust: Daniel Martí --- src/cmd/go/internal/work/exec.go | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 3ffdca5718..24e309c657 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2001,6 +2001,13 @@ func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interfa defer cleanup() cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + + // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. + // It doesn't really matter if -toolexec isn't being used. + if a != nil && a.Package != nil { + cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.ImportPath) + } + cmd.Env = append(cmd.Env, env...) start := time.Now() err := cmd.Run() -- cgit v1.2.1 From 431d58da69e8c36d654876e7808f971c5667649c Mon Sep 17 00:00:00 2001 From: Elias Naur Date: Tue, 20 Oct 2020 11:01:46 +0200 Subject: all: add GOOS=ios GOARCH=amd64 target for the ios simulator The Go toolchain has supported the simulator for years, but always in buildmode=c-archive which is intrinsically externally linked and PIE. This CL moves that support from GOOS=darwin GOARCH=amd64 -tags=ios to just GOOS=ios GOARCH=amd64 to match the change for iOS devices. This change also forces external linking and defaults to buildmode=pie to support Go binaries in the default buildmode to run on the simulator. CL 255257 added the necessary support to the exec wrapper. Updates #38485 Fixes #42100 Change-Id: I6e6ee0e8d421be53b31e3d403880e5b9b880d031 Reviewed-on: https://go-review.googlesource.com/c/go/+/263798 Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang Trust: Elias Naur --- src/cmd/go/internal/load/pkg.go | 4 +++- src/cmd/go/internal/work/init.go | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index f07bd3e075..29709a6dd3 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1966,7 +1966,9 @@ func externalLinkingForced(p *Package) bool { if cfg.BuildContext.GOARCH != "arm64" { return true } - case "darwin", "ios": + case "ios": + return true + case "darwin": if cfg.BuildContext.GOARCH == "arm64" { return true } diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index 81c4fb7465..d65c076c6a 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -168,7 +168,10 @@ func buildModeInit() { ldBuildmode = "pie" case "windows": ldBuildmode = "pie" - case "darwin", "ios": + case "ios": + codegenArg = "-shared" + ldBuildmode = "pie" + case "darwin": switch cfg.Goarch { case "arm64": codegenArg = "-shared" -- cgit v1.2.1 From 3931cc113f3f3e7d484842d6e4f53b7a78311e8e Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 22 Oct 2020 18:06:54 -0400 Subject: cmd/go: replace some more stats with fsys.Stat To support overlays For #39958 Change-Id: I5ffd72aeb7f5f30f6c60f6334a01a0a1383c7945 Reviewed-on: https://go-review.googlesource.com/c/go/+/264478 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/imports/scan.go | 3 +-- src/cmd/go/internal/modload/load.go | 3 ++- src/cmd/go/internal/modload/search.go | 2 +- src/cmd/go/internal/search/search.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/imports/scan.go b/src/cmd/go/internal/imports/scan.go index d7e674b129..ee11a8708b 100644 --- a/src/cmd/go/internal/imports/scan.go +++ b/src/cmd/go/internal/imports/scan.go @@ -7,7 +7,6 @@ package imports import ( "fmt" "io/fs" - "os" "path/filepath" "sort" "strconv" @@ -28,7 +27,7 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. if info.Mode()&fs.ModeSymlink != 0 { - info, err = os.Stat(filepath.Join(dir, name)) + info, err = fsys.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. } diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 4b3ded8326..4611fc7f6e 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -112,6 +112,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/mvs" @@ -361,7 +362,7 @@ func resolveLocalPackage(dir string) (string, error) { // If the named directory does not exist or contains no Go files, // the package does not exist. // Other errors may affect package loading, but not resolution. - if _, err := os.Stat(absDir); err != nil { + if _, err := fsys.Stat(absDir); err != nil { if os.IsNotExist(err) { // Canonicalize OS-specific errors to errDirectoryNotFound so that error // messages will be easier for users to search for. diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index 19289ceb9c..be4cb7e745 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -87,7 +87,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f if !fi.IsDir() { if fi.Mode()&fs.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index e4784e9478..57cbb282a8 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -156,7 +156,7 @@ func (m *Match) MatchPackages() { if !fi.IsDir() { if fi.Mode()&fs.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } -- cgit v1.2.1 From 75032ad8cfac4aefbacd17b47346ac8c1b5ff33f Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Thu, 8 Oct 2020 02:12:43 +0000 Subject: cmd/go: break after terminal loop condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the first time needCostly is set to true, there is no need to continue checking the remaining args. Change-Id: I07171ce50d20e2a917117a0f84c442fe978cb274 GitHub-Last-Rev: 6d0c19341b7a85d507c3ec4967bab5f83b0fad8d GitHub-Pull-Request: golang/go#41859 Reviewed-on: https://go-review.googlesource.com/c/go/+/260638 Reviewed-by: Daniel Martí Reviewed-by: Bryan C. Mills Trust: Daniel Martí Run-TryBot: Daniel Martí TryBot-Result: Go Bot --- src/cmd/go/internal/envcmd/env.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index b5a48558fa..557e418921 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -217,6 +217,7 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { needCostly = true } else { needCostly = false + checkCostly: for _, arg := range args { switch argKey(arg) { case "CGO_CFLAGS", @@ -227,6 +228,7 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { "PKG_CONFIG", "GOGCCFLAGS": needCostly = true + break checkCostly } } } -- cgit v1.2.1 From c8c3c29daa74f2d3e1a26f2e289ad3d2b9ba20dd Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 22 Oct 2020 18:26:14 -0400 Subject: cmd/go: don't import requirements into existing go.mod files Previously, if a go.mod file was present, and it only contained a module directive, any module-aware command would attempt to import requirements from a vendor configuration file like Gopkg.lock. This CL removes that functionality. It was undocumented and untested, and it can cause problems with -mod=readonly. It should never come up for go.mod files created with 'go mod init', since they have a "go" directive. For #40278 Change-Id: I64c0d67d204560aa5c775d29553883d094fd3b72 Reviewed-on: https://go-review.googlesource.com/c/go/+/264620 Trust: Jay Conrod Run-TryBot: Bryan C. Mills Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot --- src/cmd/go/internal/modload/init.go | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 1fcc53735c..9baaf41124 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -396,12 +396,6 @@ func InitMod(ctx context.Context) { base.Fatalf("go: no module declaration in go.mod.\n\tRun 'go mod edit -module=example.com/mod' to specify the module path.") } - if len(f.Syntax.Stmt) == 1 && f.Module != nil { - // Entire file is just a module statement. - // Populate require if possible. - legacyModInit() - } - if err := checkModulePathLax(f.Module.Mod.Path); err != nil { base.Fatalf("go: %v", err) } @@ -605,10 +599,6 @@ func legacyModInit() { if err := modconv.ConvertLegacyConfig(modFile, cfg, data); err != nil { base.Fatalf("go: %v", err) } - if len(modFile.Syntax.Stmt) == 1 { - // Add comment to avoid re-converting every time it runs. - modFile.AddComment("// go: no requirements found in " + name) - } return } } -- cgit v1.2.1 From d05e89a8fd35bb543df6a29faea81a85565db92f Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 22 Oct 2020 18:20:00 -0400 Subject: cmd/go: refactor modload.InitMod InitMod is split into two functions. LoadModFile parses an existing go.mod file and loads the build list (or checks vendor/modules.txt for consistency in vendor mode). CreateModFile creates a new go.mod file, possibly inferring the module path and importing a vendor configuration file. Some logic is moved from runInit to CreateModFile. init-specific logic is removed from other functions. This CL shouldn't cause substantial differences in behavior, though some error messages are slightly different. For #41712 Change-Id: Ia684945cfcf5beca30bbb81e7144fc246c4f27ed Reviewed-on: https://go-review.googlesource.com/c/go/+/264621 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 2 +- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modcmd/init.go | 17 ++--- src/cmd/go/internal/modload/buildlist.go | 2 +- src/cmd/go/internal/modload/init.go | 116 +++++++++++++++++-------------- src/cmd/go/internal/modload/load.go | 4 +- 6 files changed, 72 insertions(+), 71 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 9fd9d7446d..1c77e4d478 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -415,7 +415,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. + modload.LoadModFile(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 050a2e0e12..e2e8ba6825 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -87,7 +87,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod(ctx) // to fill Target + modload.LoadModFile(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 7cfc0e6f5b..7384f3f293 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -10,8 +10,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" "context" - "os" - "strings" ) var cmdInit = &base.Command{ @@ -33,21 +31,14 @@ func init() { } func runInit(ctx context.Context, cmd *base.Command, args []string) { - modload.CmdModInit = true if len(args) > 1 { base.Fatalf("go mod init: too many arguments") } + var modPath string if len(args) == 1 { - modload.CmdModModule = args[0] + modPath = args[0] } + modload.ForceUseModules = true - modFilePath := modload.ModFilePath() - if _, err := os.Stat(modFilePath); err == nil { - base.Fatalf("go mod init: go.mod already exists") - } - if strings.Contains(modload.CmdModModule, "@") { - base.Fatalf("go mod init: module path must not contain '@'") - } - modload.InitMod(ctx) // does all the hard work - modload.WriteGoMod() + modload.CreateModFile(ctx, modPath) // does all the hard work } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 95a68637c6..76e5fe0173 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -37,7 +37,7 @@ var buildList []module.Version // // The caller must not modify the returned list. func LoadAllModules(ctx context.Context) []module.Version { - InitMod(ctx) + LoadModFile(ctx) ReloadBuildList() WriteGoMod() return buildList diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 9baaf41124..7a8d826994 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -50,9 +50,6 @@ var ( gopath string - CmdModInit bool // running 'go mod init' - CmdModModule string // module argument for 'go mod init' - // RootMode determines whether a module root is needed. RootMode Root @@ -163,9 +160,9 @@ func Init() { os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - modRoot = base.Cwd + if modRoot != "" { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. } else if RootMode == NoRoot { if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") @@ -202,8 +199,7 @@ func Init() { base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) } - // We're in module mode. Install the hooks to make it work. - + // We're in module mode. Set any global variables that need to be set. list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") @@ -270,10 +266,6 @@ func WillBeEnabled() bool { return false } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - return true - } if modRoot := findModuleRoot(base.Cwd); modRoot == "" { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -347,16 +339,16 @@ func die() { base.Fatalf("go: cannot find main module; see 'go help modules'") } -// InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file. If InitMod is called by 'go mod init', InitMod -// will populate go.mod in memory, possibly importing dependencies from a -// legacy configuration file. For other commands, InitMod may make other -// adjustments in memory, like adding a go directive. WriteGoMod should be -// called later to write changes out to disk. +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. +// +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. WriteGoMod should be called later to +// write changes out to disk or report errors in readonly mode. // -// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not +// As a side-effect, LoadModFile sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod(ctx context.Context) { +func LoadModFile(ctx context.Context) { if len(buildList) > 0 { return } @@ -369,13 +361,6 @@ func InitMod(ctx context.Context) { return } - if CmdModInit { - // Running go mod init: do legacy module conversion - legacyModInit() - modFileToBuildList() - return - } - gomod := ModFilePath() data, err := lockedfile.Read(gomod) if err != nil { @@ -408,6 +393,50 @@ func InitMod(ctx context.Context) { } } +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot = base.Cwd + Init() + modFilePath := ModFilePath() + if _, err := os.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatalf("go: %v", err) + } + } else if err := checkModulePathLax(modPath); err != nil { + base.Fatalf("go: %v", err) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile = new(modfile.File) + modFile.AddModuleStmt(modPath) + addGoStmt() // Add the go directive before converted module requirements. + + convertedFrom, err := convertLegacyConfig(modPath) + if convertedFrom != "" { + fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + modFileToBuildList() + WriteGoMod() +} + // checkModulePathLax checks that the path meets some minimum requirements // to avoid confusing users or the module cache. The requirements are weaker // than those of module.CheckPath to allow room for weakening module path @@ -574,34 +603,23 @@ func setDefaultBuildMod() { cfg.BuildMod = "readonly" } -func legacyModInit() { - if modFile == nil { - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", path) - modFile = new(modfile.File) - modFile.AddModuleStmt(path) - addGoStmt() // Add the go directive before converted module requirements. - } - +// convertLegacyConfig imports module requirements from a legacy vendoring +// configuration file, if one is present. +func convertLegacyConfig(modPath string) (from string, err error) { for _, name := range altConfigs { cfg := filepath.Join(modRoot, name) data, err := ioutil.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] if convert == nil { - return + return "", nil } - fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(cfg)) cfg = filepath.ToSlash(cfg) - if err := modconv.ConvertLegacyConfig(modFile, cfg, data); err != nil { - base.Fatalf("go: %v", err) - } - return + err := modconv.ConvertLegacyConfig(modFile, cfg, data) + return name, err } } + return "", nil } // addGoStmt adds a go directive to the go.mod file if it does not already include one. @@ -681,14 +699,6 @@ func findAltConfig(dir string) (root, name string) { } func findModulePath(dir string) (string, error) { - if CmdModModule != "" { - // Running go mod init x/y/z; return x/y/z. - if err := module.CheckImportPath(CmdModModule); err != nil { - return "", err - } - return CmdModModule, nil - } - // TODO(bcmills): once we have located a plausible module path, we should // query version control (if available) to verify that it matches the major // version of the most recent tag. diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 4611fc7f6e..dc816540b9 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -169,7 +169,7 @@ type PackageOpts struct { // LoadPackages identifies the set of packages matching the given patterns and // loads the packages in the import graph rooted at that set. func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { - InitMod(ctx) + LoadModFile(ctx) if opts.Tags == nil { opts.Tags = imports.Tags() } @@ -494,7 +494,7 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. func ImportFromFiles(ctx context.Context, gofiles []string) { - InitMod(ctx) + LoadModFile(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) -- cgit v1.2.1 From 5f616a6fe789622f3e0ed0e8a00db9471e2a02f4 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Fri, 23 Oct 2020 11:10:10 -0400 Subject: cmd/go: in 'go mod init', suggest running 'go mod tidy' When 'go mod init' is run in an existing project, it may import requirements from a vendor configuration file, but the requirements may not be complete, and go.sum won't contain sums for module zips. With -mod=readonly, the next build command is likely to fail. 'go mod init' will now suggest running 'go mod tidy' if there are .go files or subdirectories in the current directory. We could potentially run 'go mod tidy' automatically within 'go mod init', but it seems better to guide users to using 'go mod tidy' as a separate command to fix missing dependencies. For #41712 Updates #40278 Change-Id: Iaece607f291244588a732ef4c5d576108965ca91 Reviewed-on: https://go-review.googlesource.com/c/go/+/264622 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modload/init.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 7a8d826994..f5aac4b220 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -435,6 +435,29 @@ func CreateModFile(ctx context.Context, modPath string) { modFileToBuildList() WriteGoMod() + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + fis, _ := ioutil.ReadDir(modRoot) + for _, fi := range fis { + name := fi.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || fi.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: run 'go mod tidy' to add module requirements and sums\n") + } } // checkModulePathLax checks that the path meets some minimum requirements -- cgit v1.2.1 From 5cd4390f3853b8d0d2d962f7acdac87c0eba3d77 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 13 Oct 2020 18:19:21 -0400 Subject: cmd/go: don't fetch files missing sums in readonly mode If the go command needs a .mod or .zip file in -mod=readonly mode (now the default), and that file doesn't have a hash in the main module's go.sum file, the go command will now report an error before fetching the file, rather than at the end when failing to update go.sum. The error says specifically which entry is missing. If this error is encountered when loading the build list, it will suggest 'go mod tidy'. If this error is encountered when loading a specific package (an import or command line argument), the error will mention that package and will suggest 'go mod tidy' or 'go get -d'. Fixes #41934 Fixes #41935 Change-Id: I96ec2ef9258bd4bade9915c43d47e6243c376a81 Reviewed-on: https://go-review.googlesource.com/c/go/+/262341 Reviewed-by: Bryan C. Mills Trust: Jay Conrod --- src/cmd/go/internal/load/pkg.go | 5 ++- src/cmd/go/internal/modfetch/fetch.go | 22 +++++++++++ src/cmd/go/internal/modload/import.go | 68 +++++++++++++++++++++++++++++++--- src/cmd/go/internal/modload/load.go | 2 + src/cmd/go/internal/modload/modfile.go | 14 ++++++- src/cmd/go/internal/modload/query.go | 9 +++-- src/cmd/go/internal/modload/search.go | 3 +- 7 files changed, 109 insertions(+), 14 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 29709a6dd3..fcd7728c7b 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -254,8 +254,8 @@ func (p *Package) setLoadPackageDataError(err error, path string, stk *ImportSta // package's source files themselves (scanner errors). // // TODO(matloob): Perhaps make each of those the errors in the first group - // (including modload.ImportMissingError, and the corresponding - // "cannot find package %q in any of" GOPATH-mode error + // (including modload.ImportMissingError, ImportMissingSumError, and the + // corresponding "cannot find package %q in any of" GOPATH-mode error // produced in build.(*Context).Import; modload.AmbiguousImportError, // and modload.PackageNotInModuleError; and the malformed module path errors // produced in golang.org/x/mod/module.CheckMod) implement an interface @@ -430,6 +430,7 @@ type ImportPathError interface { var ( _ ImportPathError = (*importError)(nil) _ ImportPathError = (*modload.ImportMissingError)(nil) + _ ImportPathError = (*modload.ImportMissingSumError)(nil) ) type importError struct { diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 40196c4e9a..25e9fb62c1 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -428,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error return nil } +// HaveSum returns true if the go.sum file contains an entry for mod. +// The entry's hash must be generated with a known hash algorithm. +// mod.Version may have a "/go.mod" suffix to distinguish sums for +// .mod and .zip files. +func HaveSum(mod module.Version) bool { + goSum.mu.Lock() + defer goSum.mu.Unlock() + inited, err := initGoSum() + if err != nil || !inited { + return false + } + for _, h := range goSum.m[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + return false +} + // checkMod checks the given module's checksum. func checkMod(mod module.Version) { if cfg.GOMODCACHE == "" { diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index bcbc9b0c3a..ffe8733af6 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -124,6 +124,31 @@ func (e *AmbiguousImportError) Error() string { return buf.String() } +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module in the build list contains a package, but we don't have a sum +// for its .zip file. +type ImportMissingSumError struct { + importPath string + found, inAll bool +} + +func (e *ImportMissingSumError) Error() string { + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s is provided by exactly one module", e.importPath) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s", e.importPath) + } + if e.inAll { + return message + "; try 'go mod tidy' to add it" + } + return message +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + type invalidImportError struct { importPath string err error @@ -208,13 +233,23 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di // Check each module on the build list. var dirs []string var mods []module.Version + haveSumErr := false for _, m := range buildList { if !maybeInModule(path, m.Path) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(ctx, m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. + haveSumErr = true + continue + } // Report fetch error. // Note that we don't know for sure this module is necessary, // but it certainly _could_ provide the package, and even if we @@ -230,12 +265,15 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di dirs = append(dirs, dir) } } + if len(mods) > 1 { + return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + } + if haveSumErr { + return module.Version{}, "", &ImportMissingSumError{importPath: path, found: len(mods) > 0} + } if len(mods) == 1 { return mods[0], dirs[0], nil } - if len(mods) > 0 { - return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} - } return module.Version{}, "", &ImportMissingError{Path: path, isStd: pathIsStd} } @@ -306,7 +344,8 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(ctx, m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { // Report fetch error as above. return module.Version{}, err @@ -473,9 +512,14 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // fetch downloads the given module (or its replacement) // and returns its location. // +// needSum indicates whether the module may be downloaded in readonly mode +// without a go.sum entry. It should only be false for modules fetched +// speculatively (for example, for incompatible version filtering). The sum +// will still be verified normally. +// // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { +func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -505,6 +549,18 @@ func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, e mod = r } + if cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) + } + dir, err = modfetch.Download(ctx, mod) return dir, false, err } + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion +} diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index dc816540b9..b770c19c7c 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -276,6 +276,8 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma if pkg.flags.has(pkgInAll) { if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) { imErr.inAll = true + } else if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + sumErr.inAll = true } } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 006db4f169..7a8963246b 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -406,8 +406,11 @@ type retraction struct { // taking into account any replacements for m, exclusions of its dependencies, // and/or vendoring. // -// goModSummary cannot be used on the Target module, as its requirements -// may change. +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. // // The caller must not modify the returned summary. func goModSummary(m module.Version) (*modFileSummary, error) { @@ -442,6 +445,13 @@ func goModSummary(m module.Version) (*modFileSummary, error) { if actual.Path == "" { actual = m } + if cfg.BuildMod == "readonly" && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf("; try 'go mod download %s' to add it", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } summary, err := rawGoModSummary(actual) if err != nil { return nil, err diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 6b14768388..3927051015 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -599,7 +599,8 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(ctx, r.Mod) + needSum := true + root, isLocal, err := fetch(ctx, r.Mod, needSum) if err != nil { return r, err } @@ -816,7 +817,8 @@ func (e *PackageNotInModuleError) ImportPath() string { // ModuleHasRootPackage returns whether module m contains a package m.Path. func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { - root, isLocal, err := fetch(ctx, m) + needSum := false + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { return false, err } @@ -825,7 +827,8 @@ func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { } func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { - root, _, err := fetch(ctx, m) + needSum := false + root, _, err := fetch(ctx, m, needSum) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index be4cb7e745..f6d6f5f764 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -156,7 +156,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f isLocal = true } else { var err error - root, isLocal, err = fetch(ctx, mod) + needSum := true + root, isLocal, err = fetch(ctx, mod, needSum) if err != nil { m.AddError(err) continue -- cgit v1.2.1 From 4fdb98dcb1bf7a33565c81db3c00e91a466e098d Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 15 Oct 2020 18:22:09 -0400 Subject: cmd/go: save sums for zips needed to diagnose ambiguous imports Previously, we would retain entries in go.sum for .mod files in the module graph (reachable from the main module) and for .zip files of modules providing packages. This isn't quite enough: when we load a package, we need the content of each module in the build list that *could* provide the package (that is, each module whose path is a prefix of the package's path) so we can diagnose ambiguous imports. For #33008 Change-Id: I0b4d9d68c1f4ca382f0983a3a7e537764f35c3aa Reviewed-on: https://go-review.googlesource.com/c/go/+/262781 Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills Trust: Jay Conrod --- src/cmd/go/internal/modload/init.go | 87 +++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 27 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index f5aac4b220..8fe71a2448 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -911,7 +912,10 @@ func WriteGoMod() { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). // Don't write go.mod, but write go.sum in case we added or trimmed sums. - modfetch.WriteGoSum(keepSums(true)) + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } return } @@ -924,7 +928,10 @@ func WriteGoMod() { index = indexModFile(new, modFile, false) // Update go.sum after releasing the side lock and refreshing the index. - modfetch.WriteGoSum(keepSums(true)) + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -969,41 +976,55 @@ func WriteGoMod() { // If addDirect is true, the set also includes sums for modules directly // required by go.mod, as represented by the index, with replacements applied. func keepSums(addDirect bool) map[module.Version]bool { - // Walk the module graph and keep sums needed by MVS. + // Re-derive the build list using the current list of direct requirements. + // Keep the sum for the go.mod of each visited module version (or its + // replacement). modkey := func(m module.Version) module.Version { return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} } keep := make(map[module.Version]bool) - replaced := make(map[module.Version]bool) - reqs := Reqs() - var walk func(module.Version) - walk = func(m module.Version) { - // If we build using a replacement module, keep the sum for the replacement, - // since that's the code we'll actually use during a build. - r := Replacement(m) - if r.Path == "" { - keep[modkey(m)] = true - } else { - replaced[m] = true - keep[modkey(r)] = true - } - list, _ := reqs.Required(m) - for _, r := range list { - if !keep[modkey(r)] && !replaced[r] { - walk(r) + var mu sync.Mutex + reqs := &keepSumReqs{ + Reqs: Reqs(), + visit: func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + mu.Lock() + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + keep[modkey(r)] = true } - } + mu.Unlock() + }, + } + buildList, err := mvs.BuildList(Target, reqs) + if err != nil { + panic(fmt.Sprintf("unexpected error reloading build list: %v", err)) } - walk(Target) - // Add entries for modules from which packages were loaded. + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for modules needed to + // report ambiguous import errors. We use our re-derived build list, + // since the global build list may have been tidied. if loaded != nil { - for _, pkg := range loaded.pkgs { - m := pkg.mod + actualMods := make(map[string]module.Version) + for _, m := range buildList[1:] { if r := Replacement(m); r.Path != "" { - keep[r] = true + actualMods[m.Path] = r } else { - keep[m] = true + actualMods[m.Path] = m + } + } + for _, pkg := range loaded.pkgs { + if pkg.testOf != nil || pkg.inStd { + continue + } + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if m, ok := actualMods[prefix]; ok { + keep[m] = true + } } } } @@ -1025,6 +1046,18 @@ func keepSums(addDirect bool) map[module.Version]bool { return keep } +// keepSumReqs embeds another Reqs implementation. The Required method +// calls visit for each version in the module graph. +type keepSumReqs struct { + mvs.Reqs + visit func(module.Version) +} + +func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) { + r.visit(m) + return r.Reqs.Required(m) +} + func TrimGoSum() { // Don't retain sums for direct requirements in go.mod. When TrimGoSum is // called, go.mod has not been updated, and it may contain requirements on -- cgit v1.2.1 From f24ff3856a629a6b5fefe28a1676638d5f103342 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Fri, 16 Oct 2020 14:02:16 -0400 Subject: cmd/go: change error message for missing import with unused replacement In readonly mode, if a package is not provided by any module in the build list, and there is an unused replacement that contains the package, we now recommend a 'go get' command to add a requirement on the highest replaced version. Fixes #41416 Change-Id: Iedf3539292c70ea6ba6857433fd184454d9325da Reviewed-on: https://go-review.googlesource.com/c/go/+/263146 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills Trust: Jay Conrod --- src/cmd/go/internal/modfetch/pseudo.go | 12 ++++++ src/cmd/go/internal/modload/import.go | 72 +++++++++++++++++----------------- 2 files changed, 47 insertions(+), 37 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/pseudo.go b/src/cmd/go/internal/modfetch/pseudo.go index 20c0b060ab..93eb0fad96 100644 --- a/src/cmd/go/internal/modfetch/pseudo.go +++ b/src/cmd/go/internal/modfetch/pseudo.go @@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { return v + incDecimal(patch) + "-0." + segment + build } +// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and +// revision, which may be used as a placeholder. +func ZeroPseudoVersion(major string) string { + return PseudoVersion(major, "", time.Time{}, "000000000000") +} + // incDecimal returns the decimal string incremented by 1. func incDecimal(decimal string) string { // Scan right to left turning 9s to 0s until you find a digit to increment. @@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool { return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v) } +// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, +// timestamp, and revision, as returned by ZeroPseudoVersion. +func IsZeroPseudoVersion(v string) bool { + return v == ZeroPseudoVersion(semver.Major(v)) +} + // PseudoVersionTime returns the time stamp of the pseudo-version v. // It returns an error if v is not a pseudo-version or if the time stamp // embedded in the pseudo-version is not a valid time. diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index ffe8733af6..e959347020 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -15,7 +15,6 @@ import ( "path/filepath" "sort" "strings" - "time" "cmd/go/internal/cfg" "cmd/go/internal/fsys" @@ -42,6 +41,10 @@ type ImportMissingError struct { // modules. isStd bool + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string @@ -59,6 +62,14 @@ func (e *ImportMissingError) Error() string { return "cannot find module providing package " + e.Path } + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !modfetch.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; try 'go get -d %s' to add it", e.replaced.Path, e.Path, suggestArg) + } + suggestion := "" if !HasModRoot() { suggestion = ": working directory is not part of a module" @@ -284,37 +295,6 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di // Unlike QueryPattern, queryImport prefers to add a replaced version of a // module *before* checking the proxies for a version to add. func queryImport(ctx context.Context, path string) (module.Version, error) { - pathIsStd := search.IsStandardImportPath(path) - - if cfg.BuildMod == "readonly" { - if pathIsStd { - // If the package would be in the standard library and none of the - // available replacement modules could concievably provide it, report it - // as a missing standard-library package instead of complaining that - // module lookups are disabled. - maybeReplaced := false - if index != nil { - for p := range index.highestReplaced { - if maybeInModule(path, p) { - maybeReplaced = true - break - } - } - } - if !maybeReplaced { - return module.Version{}, &ImportMissingError{Path: path, isStd: true} - } - } - - var queryErr error - if cfg.BuildModExplicit { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else if cfg.BuildModReason != "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } - return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} - } - // To avoid spurious remote fetches, try the latest replacement for each // module (golang.org/issue/26241). if index != nil { @@ -330,9 +310,9 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // used from within some other module, the user will be able to upgrade // the requirement to any real version they choose. if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { - mv = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + mv = modfetch.ZeroPseudoVersion(pathMajor[1:]) } else { - mv = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + mv = modfetch.ZeroPseudoVersion("v0") } } mods = append(mods, module.Version{Path: mp, Version: mv}) @@ -347,18 +327,23 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { needSum := true root, isLocal, err := fetch(ctx, m, needSum) if err != nil { - // Report fetch error as above. + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { return m, err } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, - // so it can only exist if in a replaced module, + // so it can only exist in a replaced module, // and we know from the above loop that it is not. return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], @@ -369,7 +354,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { } } - if pathIsStd { + if search.IsStandardImportPath(path) { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has // shimmed in via a "replace" directive. @@ -380,6 +365,19 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { return module.Version{}, &ImportMissingError{Path: path, isStd: true} } + if cfg.BuildMod == "readonly" { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} + } + // Look up module containing the package, for addition to the build list. // Goal is to determine the module, download it to dir, // and return m, dir, ImpportMissingError. -- cgit v1.2.1 From a8e2966eb01f175c330f6669f838e83af2cb73e3 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 22 Oct 2020 20:55:38 -0400 Subject: cmd/go/internal/fsys: rewrite non-idiomatic if statements https://golang.org/doc/effective_go.html#if Change-Id: I4d868e05c7827638f45b3b06d8762f5a298d56f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/264537 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/internal/fsys/fsys.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 814e323701..3275c3faf7 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -249,11 +249,11 @@ func readDir(dir string) ([]fs.FileInfo, error) { if os.IsNotExist(err) { return nil, err - } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + } + if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} - } else { - return nil, err } + return nil, err } // ReadDir provides a slice of fs.FileInfo entries corresponding @@ -267,7 +267,8 @@ func ReadDir(dir string) ([]fs.FileInfo, error) { dirNode := overlay[dir] if dirNode == nil { return readDir(dir) - } else if dirNode.isDeleted() { + } + if dirNode.isDeleted() { return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} } diskfis, err := readDir(dir) @@ -331,17 +332,18 @@ func Open(path string) (*os.File, error) { return nil, &fs.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} } return os.Open(node.actualFilePath) - } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + } + if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { // The file is deleted explicitly in the Replace map, // or implicitly because one of its parent directories was // replaced by a file. return nil, &fs.PathError{ Op: "Open", Path: path, - Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} - } else { - return os.Open(cpath) + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), + } } + return os.Open(cpath) } // IsDirWithGoFiles reports whether dir is a directory containing Go files @@ -350,7 +352,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { fis, err := ReadDir(dir) if os.IsNotExist(err) || errors.Is(err, errNotDir) { return false, nil - } else if err != nil { + } + if err != nil { return false, err } @@ -377,7 +380,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { return true, nil - } else if err != nil && firstErr == nil { + } + if err != nil && firstErr == nil { firstErr = err } } -- cgit v1.2.1 From b08dfbaa439e4e396b979e02ea2e7d36972e8b7a Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 1 Jul 2020 16:02:42 +0000 Subject: runtime,runtime/metrics: add memory metrics This change adds support for a variety of runtime memory metrics and contains the base implementation of Read for the runtime/metrics package, which lives in the runtime. It also adds testing infrastructure for the metrics package, and a bunch of format and documentation tests. For #37112. Change-Id: I16a2c4781eeeb2de0abcb045c15105f1210e2d8a Reviewed-on: https://go-review.googlesource.com/c/go/+/247041 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Reviewed-by: Michael Pratt Trust: Michael Knyszek --- src/cmd/go/internal/work/gc.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index e93031431c..0c4a7fa6e3 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -89,7 +89,11 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) if p.Standard { switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": extFiles++ } } -- cgit v1.2.1 From c305e49e96deafe54a8e43010ea76fead6da0a98 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 26 Oct 2020 14:32:13 -0400 Subject: cmd/go,cmd/compile,sync: remove special import case in cmd/go CL 253748 introduced a special case in cmd/go to allow sync to import runtime/internal/atomic. Besides introducing unnecessary complexity into cmd/go, this breaks other packages (like gopls) that understand how imports work, but don't understand this special case. Fix this by using the more standard linkname-based approach to pull the necessary functions from runtime/internal/atomic into sync. Since these are compiler intrinsics, we also have to tell the compiler that the linknamed symbols are intrinsics to get this optimization in sync. Fixes #42196. Change-Id: I1f91498c255c91583950886a89c3c9adc39a32f0 Reviewed-on: https://go-review.googlesource.com/c/go/+/265124 Trust: Austin Clements Run-TryBot: Austin Clements Reviewed-by: Bryan C. Mills Reviewed-by: Keith Randall Reviewed-by: Paul Murphy TryBot-Result: Go Bot --- src/cmd/go/internal/load/pkg.go | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index fcd7728c7b..615b5ef769 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1338,11 +1338,6 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * return p } - // Allow sync package to access lightweight atomic functions limited to the runtime. - if p.Standard && strings.HasPrefix(importerPath, "sync") && p.ImportPath == "runtime/internal/atomic" { - return p - } - // Internal is present. // Map import path back to directory corresponding to parent of internal. if i > 0 { -- cgit v1.2.1 From 1095dd6339dbaf8d7c92214396c0a4dbcfa38521 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 13 Oct 2020 10:58:13 -0400 Subject: cmd/go/internal/modload: embed PackageOpts in loaderParams Instead of duplicating PackageOpts fields in the loaderParams struct, embed the PackageOpts directly. Many of the fields are duplicated, and further fields that would also be duplicated will be added in subsequent changes. For #36460 Change-Id: I3b0770d162e901d23ec1643183eb07c413d51e0a Reviewed-on: https://go-review.googlesource.com/c/go/+/263138 Trust: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/internal/modload/buildlist.go | 4 +++- src/cmd/go/internal/modload/load.go | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 76e5fe0173..4a183d6881 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -78,7 +78,9 @@ func SetBuildList(list []module.Version) { // the build list set in SetBuildList. func ReloadBuildList() []module.Version { loaded = loadFromRoots(loaderParams{ - tags: imports.Tags(), + PackageOpts: PackageOpts{ + Tags: imports.Tags(), + }, listRoots: func() []string { return nil }, allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. }) diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index b770c19c7c..f9c468c8b2 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -250,9 +250,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } loaded = loadFromRoots(loaderParams{ - tags: opts.Tags, - loadTests: opts.LoadTests, - resolveMissing: opts.ResolveMissingImports, + PackageOpts: opts, allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, allPatternIsRoot: allPatternIsRoot, @@ -505,8 +503,10 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { } loaded = loadFromRoots(loaderParams{ - tags: tags, - resolveMissing: true, + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + }, allClosesOverTests: index.allPatternClosesOverTests(), listRoots: func() (roots []string) { roots = append(roots, imports...) @@ -659,10 +659,10 @@ type loader struct { direct map[string]bool // imported directly by main module } +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. type loaderParams struct { - tags map[string]bool // tags for scanDir - loadTests bool - resolveMissing bool + PackageOpts allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? allPatternIsRoot bool // Is the "all" pattern an additional root? @@ -821,7 +821,7 @@ func loadFromRoots(params loaderParams) *loader { ld.buildStacks() - if !ld.resolveMissing || (!HasModRoot() && !allowMissingModuleImports) { + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { // We've loaded as much as we can without resolving missing imports. break } @@ -864,7 +864,7 @@ func loadFromRoots(params loaderParams) *loader { // contributes “direct” imports — so we can't safely mark existing // dependencies as indirect-only. // Conservatively mark those dependencies as direct. - if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.tags, imports.AnyTags())) { + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true @@ -995,7 +995,7 @@ func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { // also in "all" (as above). wantTest = true - case ld.loadTests && new.has(pkgIsRoot): + case ld.LoadTests && new.has(pkgIsRoot): // LoadTest explicitly requests tests of “the root packages”. wantTest = true } @@ -1058,7 +1058,7 @@ func (ld *loader) load(pkg *loadPkg) { ld.applyPkgFlags(pkg, pkgInAll) } - imports, testImports, err := scanDir(pkg.dir, ld.tags) + imports, testImports, err := scanDir(pkg.dir, ld.Tags) if err != nil { pkg.err = err return -- cgit v1.2.1 From 69496a22682108bed606d4d509cfa3253f0cac3b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 27 Oct 2020 10:41:25 -0400 Subject: cmd/go: fix bug introduced in CL 264537 Shadowing bug noted after submit by Tom Thorogood. Change-Id: I5f40cc3863dcd7dba5469f8530e9d0460e7c3e7e Reviewed-on: https://go-review.googlesource.com/c/go/+/265537 Trust: Russ Cox Run-TryBot: Russ Cox Reviewed-by: Jay Conrod TryBot-Result: Go Bot --- src/cmd/go/internal/fsys/fsys.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 3275c3faf7..5a8b36e2bc 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -378,7 +378,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { // But it's okay if the file is a symlink pointing to a regular // file, so use os.Stat to follow symlinks and check that. actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) - if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { + fi, err := os.Stat(actualFilePath) + if err == nil && fi.Mode().IsRegular() { return true, nil } if err != nil && firstErr == nil { -- cgit v1.2.1 From d73d5d9fb0c2b963bd58ed0ab679dd71498f118e Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 27 Oct 2020 03:09:26 -0400 Subject: cmd/go/internal/imports: make Tags and AnyTags safe for concurrent use AnyTags turned up as a data race while running 'go test -race cmd/go'. I'm not sure how long the race has been present. ================== WARNING: DATA RACE Read at 0x000001141ec0 by goroutine 8: cmd/go/internal/imports.AnyTags() /usr/local/google/home/bcmills/go/src/cmd/go/internal/imports/tags.go:45 +0x10e cmd/go/internal/modload.QueryPattern.func2() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:539 +0x11d cmd/go/internal/modload.QueryPattern.func4.1() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:607 +0x3db cmd/go/internal/modload.queryPrefixModules.func1() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:677 +0xa7 Previous write at 0x000001141ec0 by goroutine 7: cmd/go/internal/imports.AnyTags() /usr/local/google/home/bcmills/go/src/cmd/go/internal/imports/tags.go:46 +0x26b cmd/go/internal/modload.QueryPattern.func2() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:539 +0x11d cmd/go/internal/modload.QueryPattern.func4.1() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:607 +0x3db cmd/go/internal/modload.queryPrefixModules.func1() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:677 +0xa7 Goroutine 8 (running) created at: cmd/go/internal/modload.queryPrefixModules() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:676 +0x284 cmd/go/internal/modload.QueryPattern.func4() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:624 +0x2e4 cmd/go/internal/modfetch.TryProxies() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modfetch/proxy.go:220 +0x107 cmd/go/internal/modload.QueryPattern() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:590 +0x69e cmd/go/internal/work.installOutsideModule() /usr/local/google/home/bcmills/go/src/cmd/go/internal/work/build.go:744 +0x4b0 cmd/go/internal/work.runInstall() /usr/local/google/home/bcmills/go/src/cmd/go/internal/work/build.go:556 +0x217 main.main() /usr/local/google/home/bcmills/go/src/cmd/go/main.go:194 +0xb94 Goroutine 7 (finished) created at: cmd/go/internal/modload.queryPrefixModules() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:676 +0x284 cmd/go/internal/modload.QueryPattern.func4() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:624 +0x2e4 cmd/go/internal/modfetch.TryProxies() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modfetch/proxy.go:220 +0x107 cmd/go/internal/modload.QueryPattern() /usr/local/google/home/bcmills/go/src/cmd/go/internal/modload/query.go:590 +0x69e cmd/go/internal/work.installOutsideModule() /usr/local/google/home/bcmills/go/src/cmd/go/internal/work/build.go:744 +0x4b0 cmd/go/internal/work.runInstall() /usr/local/google/home/bcmills/go/src/cmd/go/internal/work/build.go:556 +0x217 main.main() /usr/local/google/home/bcmills/go/src/cmd/go/main.go:194 +0xb94 ================== Change-Id: Id394978fd6ea0c30614caf8f90ee4f8e2d272843 Reviewed-on: https://go-review.googlesource.com/c/go/+/265278 Trust: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/imports/tags.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/imports/tags.go b/src/cmd/go/internal/imports/tags.go index 14b4e21a02..01b448b914 100644 --- a/src/cmd/go/internal/imports/tags.go +++ b/src/cmd/go/internal/imports/tags.go @@ -4,17 +4,23 @@ package imports -import "cmd/go/internal/cfg" +import ( + "cmd/go/internal/cfg" + "sync" +) -var tags map[string]bool +var ( + tags map[string]bool + tagsOnce sync.Once +) // Tags returns a set of build tags that are true for the target platform. // It includes GOOS, GOARCH, the compiler, possibly "cgo", // release tags like "go1.13", and user-specified build tags. func Tags() map[string]bool { - if tags == nil { + tagsOnce.Do(func() { tags = loadTags() - } + }) return tags } @@ -36,14 +42,17 @@ func loadTags() map[string]bool { return tags } -var anyTags map[string]bool +var ( + anyTags map[string]bool + anyTagsOnce sync.Once +) // AnyTags returns a special set of build tags that satisfy nearly all // build tag expressions. Only "ignore" and malformed build tag requirements // are considered false. func AnyTags() map[string]bool { - if anyTags == nil { + anyTagsOnce.Do(func() { anyTags = map[string]bool{"*": true} - } + }) return anyTags } -- cgit v1.2.1 From 3c55aea67aa65c62016020d5907b481da010f7e0 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 22 Oct 2020 21:33:02 -0400 Subject: cmd/go/internal/fsys: add Glob Glob is needed for //go:embed processing. Also change TestReadDir to be deterministic and print more output about failures. Change-Id: Ie22a9c5b32bda753579ff98cec1d28e3244c4e06 Reviewed-on: https://go-review.googlesource.com/c/go/+/264538 Trust: Russ Cox Trust: Jay Conrod Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Michael Matloob --- src/cmd/go/internal/fsys/fsys.go | 163 +++++++++++++++++++++ src/cmd/go/internal/fsys/fsys_test.go | 258 +++++++++++++++++++++++++--------- 2 files changed, 353 insertions(+), 68 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 5a8b36e2bc..44d9b1368b 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "sort" "strings" "time" @@ -514,3 +515,165 @@ func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } func (f fakeDir) IsDir() bool { return true } func (f fakeDir) Sys() interface{} { return nil } + +// Glob is like filepath.Glob but uses the overlay file system. +func Glob(pattern string) (matches []string, err error) { + // Check pattern is well-formed. + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, err + } + if !hasMeta(pattern) { + if _, err = lstat(pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + volumeLen := 0 + if runtime.GOOS == "windows" { + volumeLen, dir = cleanGlobPathWindows(dir) + } else { + dir = cleanGlobPath(dir) + } + + if !hasMeta(dir[volumeLen:]) { + return glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, filepath.ErrBadPattern + } + + var m []string + m, err = Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(filepath.Separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +func volumeNameLen(path string) int { + isSlash := func(c uint8) bool { + return c == '\\' || c == '/' + } + if len(path) < 2 { + return 0 + } + // with drive letter + c := path[0] + if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return 2 + } + // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return n + } + break + } + } + } + return 0 +} + +// cleanGlobPathWindows is windows version of cleanGlobPath. +func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { + vollen := volumeNameLen(path) + switch { + case path == "": + return 0, "." + case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ + // do nothing to the path + return vollen + 1, path + case vollen == len(path) && len(path) == 2: // C: + return vollen, path + "." // convert C: into C:. + default: + if vollen >= len(path) { + vollen = len(path) - 1 + } + return vollen, path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := Stat(dir) + if err != nil { + return // ignore I/O error + } + if !fi.IsDir() { + return // ignore I/O error + } + + list, err := ReadDir(dir) + if err != nil { + return // ignore I/O error + } + + var names []string + for _, info := range list { + names = append(names, info.Name()) + } + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by filepath.Match. +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index fd98d13f3d..22ad2fe445 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -152,8 +152,7 @@ six } } -func TestReadDir(t *testing.T) { - initOverlay(t, ` +const readDirOverlay = ` { "Replace": { "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", @@ -210,70 +209,124 @@ x -- overlayfiles/this_is_a_directory/file.txt -- -- overlayfiles/textfile_txt_file.go -- x -`) +` + +func TestReadDir(t *testing.T) { + initOverlay(t, readDirOverlay) - testCases := map[string][]struct { + type entry struct { name string size int64 isDir bool + } + + testCases := []struct { + dir string + want []entry }{ - ".": { - {"other", 0, true}, - {"overlayfiles", 0, true}, - {"parentoverwritten", 0, true}, - {"subdir1", 0, true}, - {"subdir10", 0, true}, - {"subdir11", 0, false}, - {"subdir2", 0, true}, - {"subdir3", 0, true}, - {"subdir4", 2, false}, - // no subdir5. - {"subdir6", 0, true}, - {"subdir7", 0, true}, - {"subdir8", 0, true}, - {"subdir9", 0, true}, - {"textfile.txt", 0, true}, - }, - "subdir1": {{"file1.txt", 1, false}}, - "subdir2": {{"file2.txt", 2, false}}, - "subdir3": {{"file3a.txt", 3, false}, {"file3b.txt", 6, false}}, - "subdir6": { - {"anothersubsubdir", 0, true}, - {"asubsubdir", 0, true}, - {"file.txt", 0, false}, - {"zsubsubdir", 0, true}, - }, - "subdir6/asubsubdir": {{"afile.txt", 0, false}, {"file.txt", 0, false}, {"zfile.txt", 0, false}}, - "subdir8": {{"doesntexist", 0, false}}, // entry is returned even if destination file doesn't exist - // check that read dir actually redirects files that already exist - // the original this_file_is_overlaid.txt is empty - "subdir9": {{"this_file_is_overlaid.txt", 9, false}}, - "subdir10": {}, - "parentoverwritten": {{"subdir1", 2, false}}, - "textfile.txt": {{"file.go", 2, false}}, - } - - for dir, want := range testCases { - fis, err := ReadDir(dir) + { + ".", []entry{ + {"other", 0, true}, + {"overlayfiles", 0, true}, + {"parentoverwritten", 0, true}, + {"subdir1", 0, true}, + {"subdir10", 0, true}, + {"subdir11", 0, false}, + {"subdir2", 0, true}, + {"subdir3", 0, true}, + {"subdir4", 2, false}, + // no subdir5. + {"subdir6", 0, true}, + {"subdir7", 0, true}, + {"subdir8", 0, true}, + {"subdir9", 0, true}, + {"textfile.txt", 0, true}, + }, + }, + { + "subdir1", []entry{ + {"file1.txt", 1, false}, + }, + }, + { + "subdir2", []entry{ + {"file2.txt", 2, false}, + }, + }, + { + "subdir3", []entry{ + {"file3a.txt", 3, false}, + {"file3b.txt", 6, false}, + }, + }, + { + "subdir6", []entry{ + {"anothersubsubdir", 0, true}, + {"asubsubdir", 0, true}, + {"file.txt", 0, false}, + {"zsubsubdir", 0, true}, + }, + }, + { + "subdir6/asubsubdir", []entry{ + {"afile.txt", 0, false}, + {"file.txt", 0, false}, + {"zfile.txt", 0, false}, + }, + }, + { + "subdir8", []entry{ + {"doesntexist", 0, false}, // entry is returned even if destination file doesn't exist + }, + }, + { + // check that read dir actually redirects files that already exist + // the original this_file_is_overlaid.txt is empty + "subdir9", []entry{ + {"this_file_is_overlaid.txt", 9, false}, + }, + }, + { + "subdir10", []entry{}, + }, + { + "parentoverwritten", []entry{ + {"subdir1", 2, false}, + }, + }, + { + "textfile.txt", []entry{ + {"file.go", 2, false}, + }, + }, + } + + for _, tc := range testCases { + dir, want := tc.dir, tc.want + infos, err := ReadDir(dir) if err != nil { - t.Fatalf("ReadDir(%q): got error %q, want no error", dir, err) - } - if len(fis) != len(want) { - t.Fatalf("ReadDir(%q) result: got %v entries; want %v entries", dir, len(fis), len(want)) + t.Errorf("ReadDir(%q): %v", dir, err) + continue } - for i := range fis { - if fis[i].Name() != want[i].name { - t.Fatalf("ReadDir(%q) entry %v: got Name() = %v, want %v", dir, i, fis[i].Name(), want[i].name) - } - if fis[i].IsDir() != want[i].isDir { - t.Fatalf("ReadDir(%q) entry %v: got IsDir() = %v, want %v", dir, i, fis[i].IsDir(), want[i].isDir) - } - if want[i].isDir { - // We don't try to get size right for directories - continue - } - if fis[i].Size() != want[i].size { - t.Fatalf("ReadDir(%q) entry %v: got Size() = %v, want %v", dir, i, fis[i].Size(), want[i].size) + // Sorted diff of want and infos. + for len(infos) > 0 || len(want) > 0 { + switch { + case len(want) == 0 || len(infos) > 0 && infos[0].Name() < want[0].name: + t.Errorf("ReadDir(%q): unexpected entry: %s IsDir=%v Size=%v", dir, infos[0].Name(), infos[0].IsDir(), infos[0].Size()) + infos = infos[1:] + case len(infos) == 0 || len(want) > 0 && want[0].name < infos[0].Name(): + t.Errorf("ReadDir(%q): missing entry: %s IsDir=%v Size=%v", dir, want[0].name, want[0].isDir, want[0].size) + want = want[1:] + default: + infoSize := infos[0].Size() + if want[0].isDir { + infoSize = 0 + } + if infos[0].IsDir() != want[0].isDir || want[0].isDir && infoSize != want[0].size { + t.Errorf("ReadDir(%q): %s: IsDir=%v Size=%v, want IsDir=%v Size=%v", dir, want[0].name, infos[0].IsDir(), infoSize, want[0].isDir, want[0].size) + } + infos = infos[1:] + want = want[1:] } } } @@ -290,11 +343,80 @@ x } for _, dir := range errCases { - _, gotErr := ReadDir(dir) - if gotErr == nil { - t.Errorf("ReadDir(%q): got no error, want error", dir) - } else if _, ok := gotErr.(*fs.PathError); !ok { - t.Errorf("ReadDir(%q): got error with string %q and type %T, want fs.PathError", dir, gotErr.Error(), gotErr) + _, err := ReadDir(dir) + if _, ok := err.(*fs.PathError); !ok { + t.Errorf("ReadDir(%q): err = %T (%v), want fs.PathError", dir, err, err) + } + } +} + +func TestGlob(t *testing.T) { + initOverlay(t, readDirOverlay) + + testCases := []struct { + pattern string + match []string + }{ + { + "*o*", + []string{ + "other", + "overlayfiles", + "parentoverwritten", + }, + }, + { + "subdir2/file2.txt", + []string{ + "subdir2/file2.txt", + }, + }, + { + "*/*.txt", + []string{ + "overlayfiles/subdir2_file2.txt", + "overlayfiles/subdir3_file3b.txt", + "overlayfiles/subdir6_asubsubdir_afile.txt", + "overlayfiles/subdir6_asubsubdir_zfile.txt", + "overlayfiles/subdir6_zsubsubdir_file.txt", + "overlayfiles/subdir7_asubsubdir_file.txt", + "overlayfiles/subdir7_zsubsubdir_file.txt", + "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir1/file1.txt", + "subdir2/file2.txt", + "subdir3/file3a.txt", + "subdir3/file3b.txt", + "subdir6/file.txt", + "subdir9/this_file_is_overlaid.txt", + }, + }, + } + + for _, tc := range testCases { + pattern := tc.pattern + match, err := Glob(pattern) + if err != nil { + t.Errorf("Glob(%q): %v", pattern, err) + continue + } + want := tc.match + for i, name := range want { + if name != tc.pattern { + want[i] = filepath.FromSlash(name) + } + } + for len(match) > 0 || len(want) > 0 { + switch { + case len(match) == 0 || len(want) > 0 && want[0] < match[0]: + t.Errorf("Glob(%q): missing match: %s", pattern, want[0]) + want = want[1:] + case len(want) == 0 || len(match) > 0 && match[0] < want[0]: + t.Errorf("Glob(%q): extra match: %s", pattern, match[0]) + match = match[1:] + default: + want = want[1:] + match = match[1:] + } } } } @@ -605,7 +727,7 @@ contents of other file } } -func TestWalk_SkipDir(t *testing.T) { +func TestWalkSkipDir(t *testing.T) { initOverlay(t, ` { "Replace": { @@ -639,7 +761,7 @@ func TestWalk_SkipDir(t *testing.T) { } } -func TestWalk_Error(t *testing.T) { +func TestWalkError(t *testing.T) { initOverlay(t, "{}") alreadyCalled := false @@ -662,7 +784,7 @@ func TestWalk_Error(t *testing.T) { } } -func TestWalk_Symlink(t *testing.T) { +func TestWalkSymlink(t *testing.T) { testenv.MustHaveSymlink(t) initOverlay(t, `{ @@ -942,7 +1064,7 @@ contents`, } } -func TestStat_Symlink(t *testing.T) { +func TestStatSymlink(t *testing.T) { testenv.MustHaveSymlink(t) initOverlay(t, `{ -- cgit v1.2.1 From 9113d8c37f9f40ab86b12bddb98dee2b1c0a344f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 22 Oct 2020 16:30:26 +0100 Subject: doc/go1.16: document BuildID in 'go list -export' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This corresponds to the feature in https://golang.org/cl/263542, since this can be a noteworthy change to people writing tools to inspect Go builds. Also amend the wording to clarify that build IDs are for an entire compiled package, not just their export data or object file. Change-Id: I2eb295492807d5d2997a35e5e2371914cb3ad3a0 Reviewed-on: https://go-review.googlesource.com/c/go/+/264158 Trust: Daniel Martí Run-TryBot: Daniel Martí Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 2 +- src/cmd/go/internal/load/pkg.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 1c77e4d478..89088f5def 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -66,7 +66,7 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is: BinaryOnly bool // binary-only package (no longer supported) ForTest string // package is only for use in named test Export string // file containing export data (when using -export) - BuildID string // build ID of the export data (when using -export) + BuildID string // build ID of the compiled package (when using -export) Module *Module // info about package's containing module, if any (can be nil) Match []string // command-line patterns matching this package DepOnly bool // package is only a dependency, not explicitly listed diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 615b5ef769..4c541b9017 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -61,7 +61,7 @@ type PackagePublic struct { ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory ForTest string `json:",omitempty"` // package is only for use in named test Export string `json:",omitempty"` // file containing export data (set by go list -export) - BuildID string `json:",omitempty"` // build ID of the export data (set by go list -export) + BuildID string `json:",omitempty"` // build ID of the compiled package (set by go list -export) Module *modinfo.ModulePublic `json:",omitempty"` // info about package's module, if any Match []string `json:",omitempty"` // command-line patterns matching this package Goroot bool `json:",omitempty"` // is this package found in the Go root? -- cgit v1.2.1 From 94f3762462a999bfc5491c8d1b892110651e23d6 Mon Sep 17 00:00:00 2001 From: Dave Pifke Date: Thu, 2 Jul 2020 00:48:37 +0000 Subject: cmd/go: add -include to cgo whitelist Fixes #39988. Change-Id: Ia6f5b73e6508f27e3badbcbd29dbeadffd55a932 Reviewed-on: https://go-review.googlesource.com/c/go/+/240739 Trust: Tobias Klauser Run-TryBot: Tobias Klauser TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/work/security.go | 1 + src/cmd/go/internal/work/security_test.go | 3 +++ 2 files changed, 4 insertions(+) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index d2a2697f0f..bcc29c8cbe 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -132,6 +132,7 @@ var validCompilerFlagsWithNextArg = []string{ "-U", "-I", "-framework", + "-include", "-isysroot", "-isystem", "--sysroot", diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go index 11e74f29c6..43a0ab1e47 100644 --- a/src/cmd/go/internal/work/security_test.go +++ b/src/cmd/go/internal/work/security_test.go @@ -62,6 +62,8 @@ var goodCompilerFlags = [][]string{ {"-I", "=/usr/include/libxml2"}, {"-I", "dir"}, {"-I", "$SYSROOT/dir"}, + {"-isystem", "/usr/include/mozjs-68"}, + {"-include", "/usr/include/mozjs-68/RequiredDefines.h"}, {"-framework", "Chocolate"}, {"-x", "c"}, {"-v"}, @@ -91,6 +93,7 @@ var badCompilerFlags = [][]string{ {"-I", "@foo"}, {"-I", "-foo"}, {"-I", "=@obj"}, + {"-include", "@foo"}, {"-framework", "-Caffeine"}, {"-framework", "@Home"}, {"-x", "--c"}, -- cgit v1.2.1 From cf6cfba4d5358404dd890f6025e573a4b2156543 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 28 Oct 2020 12:07:31 +0100 Subject: cmd/go/internal/modfetch: update expected tags for TestCodeRepoVersions Updates #28856 Change-Id: I89c564cefd7c5777904bc00c74617dab693373bf Reviewed-on: https://go-review.googlesource.com/c/go/+/265819 Trust: Tobias Klauser Run-TryBot: Tobias Klauser Reviewed-by: Alberto Donizetti Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot --- src/cmd/go/internal/modfetch/coderepo_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd/go/internal') diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go index 28c5e67a28..4364fef6d1 100644 --- a/src/cmd/go/internal/modfetch/coderepo_test.go +++ b/src/cmd/go/internal/modfetch/coderepo_test.go @@ -648,7 +648,7 @@ var codeRepoVersionsTests = []struct { { vcs: "git", path: "gopkg.in/russross/blackfriday.v2", - versions: []string{"v2.0.0", "v2.0.1"}, + versions: []string{"v2.0.0", "v2.0.1", "v2.1.0-pre.1"}, }, { vcs: "git", -- cgit v1.2.1