diff options
Diffstat (limited to 'src/cmd/go/internal')
77 files changed, 3394 insertions, 962 deletions
diff --git a/src/cmd/go/internal/base/goflags.go b/src/cmd/go/internal/base/goflags.go index 4da27550fd..267006be7a 100644 --- a/src/cmd/go/internal/base/goflags.go +++ b/src/cmd/go/internal/base/goflags.go @@ -92,7 +92,11 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } for _, goflag := range goflags { name, value, hasValue := goflag, "", false - if i := strings.Index(goflag, "="); i >= 0 { + // Ignore invalid flags like '=' or '=value'. + // If it is not reported in InitGOFlags it means we don't want to report it. + if i := strings.Index(goflag, "="); i == 0 { + continue + } else if i > 0 { name, value, hasValue = goflag[:i], goflag[i+1:], true } if strings.HasPrefix(name, "--") { diff --git a/src/cmd/go/internal/cache/cache.go b/src/cmd/go/internal/cache/cache.go index 15545ac31f..5464fe5685 100644 --- a/src/cmd/go/internal/cache/cache.go +++ b/src/cmd/go/internal/cache/cache.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -54,7 +55,7 @@ func Open(dir string) (*Cache, error) { return nil, err } if !info.IsDir() { - return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 9bf1db73ef..67d581f6e6 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -11,6 +11,7 @@ import ( "fmt" "go/build" "internal/cfg" + "io" "io/ioutil" "os" "path/filepath" @@ -18,6 +19,8 @@ import ( "strings" "sync" + "cmd/go/internal/fsys" + "cmd/internal/objabi" ) @@ -104,6 +107,15 @@ func defaultContext() build.Context { // Nothing to do here. } + ctxt.OpenFile = func(path string) (io.ReadCloser, error) { + return fsys.Open(path) + } + ctxt.ReadDir = fsys.ReadDir + ctxt.IsDir = func(path string) bool { + isDir, err := fsys.IsDir(path) + return err == nil && isDir + } + return ctxt } diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 6bfd7ae21e..87933f04f3 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -8,6 +8,7 @@ package clean import ( "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -172,7 +173,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { f, err := lockedfile.Edit(filepath.Join(dir, "testexpire.txt")) if err == nil { now := time.Now().UnixNano() - buf, _ := ioutil.ReadAll(f) + buf, _ := io.ReadAll(f) prev, _ := strconv.ParseInt(strings.TrimSpace(string(buf)), 10, 64) if now > prev { if err = f.Truncate(0); err == nil { @@ -275,6 +276,8 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem, elem+".exe", + p.DefaultExecName(), + p.DefaultExecName()+".exe", ) } @@ -282,16 +285,28 @@ func clean(p *load.Package) { allRemove = append(allRemove, elem+".test", elem+".test.exe", + p.DefaultExecName()+".test", + p.DefaultExecName()+".test.exe", ) - // Remove a potential executable for each .go file in the directory that + // Remove a potential executable, test executable for each .go file in the directory that // is not part of the directory's package. for _, dir := range dirs { name := dir.Name() if packageFile[name] { continue } - if !dir.IsDir() && strings.HasSuffix(name, ".go") { + + if dir.IsDir() { + continue + } + + if strings.HasSuffix(name, "_test.go") { + base := name[:len(name)-len("_test.go")] + allRemove = append(allRemove, base+".test", base+".test.exe") + } + + if strings.HasSuffix(name, ".go") { // TODO(adg,rsc): check that this .go file is actually // in "package main", and therefore capable of building // to an executable file. diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 7bd75f7305..557e418921 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -21,6 +21,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cache" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" @@ -197,12 +198,26 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { env := cfg.CmdEnv env = append(env, ExtraEnvVars()...) + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Do we need to call ExtraEnvVarsCostly, which is a bit expensive? - // Only if we're listing all environment variables ("go env") - // or the variables being requested are in the extra list. - needCostly := true - if len(args) > 0 { + needCostly := false + if *envU || *envW { + // We're overwriting or removing default settings, + // so it doesn't really matter what the existing settings are. + // + // Moreover, we haven't validated the new settings yet, so it is + // important that we NOT perform any actions based on them, + // such as initializing the builder to compute other variables. + } else if len(args) == 0 { + // We're listing all environment variables ("go env"), + // including the expensive ones. + needCostly = true + } else { needCostly = false + checkCostly: for _, arg := range args { switch argKey(arg) { case "CGO_CFLAGS", @@ -213,6 +228,7 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { "PKG_CONFIG", "GOGCCFLAGS": needCostly = true + break checkCostly } } } @@ -264,6 +280,13 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { } } + gotmp, okGOTMP := add["GOTMPDIR"] + if okGOTMP { + if !filepath.IsAbs(gotmp) && gotmp != "" { + base.Fatalf("go env -w: GOTMPDIR must be an absolute path") + } + } + updateEnvFile(add, nil) return } @@ -403,6 +426,11 @@ func checkEnvWrite(key, val string) error { if !filepath.IsAbs(val) && val != "" { return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val) } + // Make sure CC and CXX are absolute paths + case "CC", "CXX": + if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) { + return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val) + } } if !utf8.ValidString(val) { @@ -497,7 +525,10 @@ func lineToKey(line string) string { } // sortKeyValues sorts a sequence of lines by key. -// It differs from sort.Strings in that GO386= sorts after GO=. +// It differs from sort.Strings in that keys which are GOx where x is an ASCII +// character smaller than = sort after GO=. +// (There are no such keys currently. It used to matter for GO386 which was +// removed in Go 1.16.) func sortKeyValues(lines []string) { sort.Slice(lines, func(i, j int) bool { return lineToKey(lines[i]) < lineToKey(lines[j]) diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go new file mode 100644 index 0000000000..44d9b1368b --- /dev/null +++ b/src/cmd/go/internal/fsys/fsys.go @@ -0,0 +1,679 @@ +// Package fsys is an abstraction for reading files that +// allows for virtual overlays on top of the files on disk. +package fsys + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// OverlayFile is the path to a text file in the OverlayJSON format. +// It is the value of the -overlay flag. +var OverlayFile string + +// OverlayJSON is the format overlay files are expected to be in. +// The Replace map maps from overlaid paths to replacement paths: +// the Go command will forward all reads trying to open +// each overlaid path to its replacement path, or consider the overlaid +// path not to exist if the replacement path is empty. +type OverlayJSON struct { + Replace map[string]string +} + +type node struct { + actualFilePath string // empty if a directory + children map[string]*node // path element → file or directory +} + +func (n *node) isDir() bool { + return n.actualFilePath == "" && n.children != nil +} + +func (n *node) isDeleted() bool { + return n.actualFilePath == "" && n.children == nil +} + +// TODO(matloob): encapsulate these in an io/fs-like interface +var overlay map[string]*node // path -> file or directory node +var cwd string // copy of base.Cwd to avoid dependency + +// Canonicalize a path for looking it up in the overlay. +// Important: filepath.Join(cwd, path) doesn't always produce +// the correct absolute path if path is relative, because on +// Windows producing the correct absolute path requires making +// a syscall. So this should only be used when looking up paths +// in the overlay, or canonicalizing the paths in the overlay. +func canonicalize(path string) string { + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { + // On Windows filepath.Join(cwd, path) doesn't always work. In general + // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go + // use filepath.Join(cwd, path), but cmd/go specifically supports Windows + // paths that start with "\" which implies the path is relative to the + // volume of the working directory. See golang.org/issue/8130. + return filepath.Join(v, path) + } + + // Make the path absolute. + return filepath.Join(cwd, path) +} + +// Init initializes the overlay, if one is being used. +func Init(wd string) error { + if overlay != nil { + // already initialized + return nil + } + + cwd = wd + + if OverlayFile == "" { + return nil + } + + b, err := ioutil.ReadFile(OverlayFile) + if err != nil { + return fmt.Errorf("reading overlay file: %v", err) + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(b, &overlayJSON); err != nil { + return fmt.Errorf("parsing overlay JSON: %v", err) + } + + return initFromJSON(overlayJSON) +} + +func initFromJSON(overlayJSON OverlayJSON) error { + // Canonicalize the paths in in the overlay map. + // Use reverseCanonicalized to check for collisions: + // no two 'from' paths should canonicalize to the same path. + overlay = make(map[string]*node) + reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates + // Build a table of file and directory nodes from the replacement map. + + // Remove any potential non-determinism from iterating over map by sorting it. + replaceFrom := make([]string, 0, len(overlayJSON.Replace)) + for k := range overlayJSON.Replace { + replaceFrom = append(replaceFrom, k) + } + sort.Strings(replaceFrom) + + for _, from := range replaceFrom { + to := overlayJSON.Replace[from] + // Canonicalize paths and check for a collision. + if from == "" { + return fmt.Errorf("empty string key in overlay file Replace map") + } + cfrom := canonicalize(from) + if to != "" { + // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". + to = canonicalize(to) + } + if otherFrom, seen := reverseCanonicalized[cfrom]; seen { + return fmt.Errorf( + "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) + } + reverseCanonicalized[cfrom] = from + from = cfrom + + // Create node for overlaid file. + dir, base := filepath.Dir(from), filepath.Base(from) + if n, ok := overlay[from]; ok { + // All 'from' paths in the overlay are file paths. Since the from paths + // are in a map, they are unique, so if the node already exists we added + // it below when we create parent directory nodes. That is, that + // both a file and a path to one of its parent directories exist as keys + // in the Replace map. + // + // This only applies if the overlay directory has any files or directories + // in it: placeholder directories that only contain deleted files don't + // count. They are safe to be overwritten with actual files. + for _, f := range n.children { + if !f.isDeleted() { + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) + } + } + } + overlay[from] = &node{actualFilePath: to} + + // Add parent directory nodes to overlay structure. + childNode := overlay[from] + for { + dirNode := overlay[dir] + if dirNode == nil || dirNode.isDeleted() { + dirNode = &node{children: make(map[string]*node)} + overlay[dir] = dirNode + } + if childNode.isDeleted() { + // Only create one parent for a deleted file: + // the directory only conditionally exists if + // there are any non-deleted children, so + // we don't create their parents. + if dirNode.isDir() { + dirNode.children[base] = childNode + } + break + } + if !dirNode.isDir() { + // This path already exists as a file, so it can't be a parent + // directory. See comment at error above. + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) + } + dirNode.children[base] = childNode + parent := filepath.Dir(dir) + if parent == dir { + break // reached the top; there is no parent + } + dir, base = parent, filepath.Base(dir) + childNode = dirNode + } + } + + return nil +} + +// IsDir returns true if path is a directory on disk or in the +// overlay. +func IsDir(path string) (bool, error) { + path = canonicalize(path) + + if _, ok := parentIsOverlayFile(path); ok { + return false, nil + } + + if n, ok := overlay[path]; ok { + return n.isDir(), nil + } + + fi, err := os.Stat(path) + if err != nil { + return false, err + } + + return fi.IsDir(), nil +} + +// parentIsOverlayFile returns whether name or any of +// its parents are files in the overlay, and the first parent found, +// including name itself, that's a file in the overlay. +func parentIsOverlayFile(name string) (string, bool) { + if overlay != nil { + // Check if name can't possibly be a directory because + // it or one of its parents is overlaid with a file. + // TODO(matloob): Maybe save this to avoid doing it every time? + prefix := name + for { + node := overlay[prefix] + if node != nil && !node.isDir() { + return prefix, true + } + parent := filepath.Dir(prefix) + if parent == prefix { + break + } + prefix = parent + } + } + + return "", false +} + +// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles +// that the argument is not a directory, so that IsDirWithGoFiles doesn't +// return an error. +var errNotDir = errors.New("not a directory") + +// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. +// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory +// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). +func readDir(dir string) ([]fs.FileInfo, error) { + fis, err := ioutil.ReadDir(dir) + if err == nil { + return fis, nil + } + + if os.IsNotExist(err) { + return nil, err + } + if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + return nil, err +} + +// ReadDir provides a slice of fs.FileInfo entries corresponding +// to the overlaid files in the directory. +func ReadDir(dir string) ([]fs.FileInfo, error) { + dir = canonicalize(dir) + if _, ok := parentIsOverlayFile(dir); ok { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + + dirNode := overlay[dir] + if dirNode == nil { + return readDir(dir) + } + if dirNode.isDeleted() { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} + } + diskfis, err := readDir(dir) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { + return nil, err + } + + // Stat files in overlay to make composite list of fileinfos + files := make(map[string]fs.FileInfo) + for _, f := range diskfis { + files[f.Name()] = f + } + for name, to := range dirNode.children { + switch { + case to.isDir(): + files[name] = fakeDir(name) + case to.isDeleted(): + delete(files, name) + default: + // This is a regular file. + f, err := os.Lstat(to.actualFilePath) + if err != nil { + files[name] = missingFile(name) + continue + } else if f.IsDir() { + return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", + filepath.Join(dir, name), to.actualFilePath) + } + // Add a fileinfo for the overlaid file, so that it has + // the original file's name, but the overlaid file's metadata. + files[name] = fakeFile{name, f} + } + } + sortedFiles := diskfis[:0] + for _, f := range files { + sortedFiles = append(sortedFiles, f) + } + sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) + return sortedFiles, nil +} + +// OverlayPath returns the path to the overlaid contents of the +// file, the empty string if the overlay deletes the file, or path +// itself if the file is not in the overlay, the file is a directory +// in the overlay, or there is no overlay. +// It returns true if the path is overlaid with a regular file +// or deleted, and false otherwise. +func OverlayPath(path string) (string, bool) { + if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { + return p.actualFilePath, ok + } + + return path, false +} + +// Open opens the file at or overlaid on the given path. +func Open(path string) (*os.File, error) { + cpath := canonicalize(path) + if node, ok := overlay[cpath]; ok { + if node.isDir() { + return nil, &fs.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} + } + return os.Open(node.actualFilePath) + } + if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + // The file is deleted explicitly in the Replace map, + // or implicitly because one of its parent directories was + // replaced by a file. + return nil, &fs.PathError{ + Op: "Open", + Path: path, + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), + } + } + return os.Open(cpath) +} + +// IsDirWithGoFiles reports whether dir is a directory containing Go files +// either on disk or in the overlay. +func IsDirWithGoFiles(dir string) (bool, error) { + fis, err := ReadDir(dir) + if os.IsNotExist(err) || errors.Is(err, errNotDir) { + return false, nil + } + if err != nil { + return false, err + } + + var firstErr error + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // TODO(matloob): this enforces that the "from" in the map + // has a .go suffix, but the actual destination file + // doesn't need to have a .go suffix. Is this okay with the + // compiler? + if !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if fi.Mode().IsRegular() { + return true, nil + } + + // fi is the result of an Lstat, so it doesn't follow symlinks. + // But it's okay if the file is a symlink pointing to a regular + // file, so use os.Stat to follow symlinks and check that. + actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) + fi, err := os.Stat(actualFilePath) + if err == nil && fi.Mode().IsRegular() { + return true, nil + } + if err != nil && firstErr == nil { + firstErr = err + } + } + + // No go files found in directory. + return false, firstErr +} + +// walk recursively descends path, calling walkFn. Copied, with some +// modifications from path/filepath.walk. +func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + fis, readErr := ReadDir(path) + walkErr := walkFn(path, info, readErr) + // If readErr != nil, walk can't walk into this directory. + // walkErr != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of readErr and walkErr isn't nil, walk will return. + if readErr != nil || walkErr != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore readErr and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return walkErr + } + + for _, fi := range fis { + filename := filepath.Join(path, fi.Name()) + if walkErr = walk(filename, fi, walkFn); walkErr != nil { + if !fi.IsDir() || walkErr != filepath.SkipDir { + return walkErr + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + info, err := lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} + +// lstat implements a version of os.Lstat that operates on the overlay filesystem. +func lstat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Lstat, "lstat") +} + +// Stat implements a version of os.Stat that operates on the overlay filesystem. +func Stat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Stat, "stat") +} + +// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). +func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { + cpath := canonicalize(path) + + if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} + } + + node, ok := overlay[cpath] + if !ok { + // The file or directory is not overlaid. + return osStat(path) + } + + switch { + case node.isDeleted(): + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} + case node.isDir(): + return fakeDir(filepath.Base(path)), nil + default: + fi, err := osStat(node.actualFilePath) + if err != nil { + return nil, err + } + return fakeFile{name: filepath.Base(path), real: fi}, nil + } +} + +// fakeFile provides an fs.FileInfo implementation for an overlaid file, +// so that the file has the name of the overlaid file, but takes all +// other characteristics of the replacement file. +type fakeFile struct { + name string + real fs.FileInfo +} + +func (f fakeFile) Name() string { return f.name } +func (f fakeFile) Size() int64 { return f.real.Size() } +func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } +func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } +func (f fakeFile) IsDir() bool { return f.real.IsDir() } +func (f fakeFile) Sys() interface{} { return f.real.Sys() } + +// missingFile provides an fs.FileInfo for an overlaid file where the +// destination file in the overlay doesn't exist. It returns zero values +// for the fileInfo methods other than Name, set to the file's name, and Mode +// set to ModeIrregular. +type missingFile string + +func (f missingFile) Name() string { return string(f) } +func (f missingFile) Size() int64 { return 0 } +func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } +func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } +func (f missingFile) IsDir() bool { return false } +func (f missingFile) Sys() interface{} { return nil } + +// fakeDir provides an fs.FileInfo implementation for directories that are +// implicitly created by overlaid files. Each directory in the +// path of an overlaid file is considered to exist in the overlay filesystem. +type fakeDir string + +func (f fakeDir) Name() string { return string(f) } +func (f fakeDir) Size() int64 { return 0 } +func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } +func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } +func (f fakeDir) IsDir() bool { return true } +func (f fakeDir) Sys() interface{} { return nil } + +// Glob is like filepath.Glob but uses the overlay file system. +func Glob(pattern string) (matches []string, err error) { + // Check pattern is well-formed. + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, err + } + if !hasMeta(pattern) { + if _, err = lstat(pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + volumeLen := 0 + if runtime.GOOS == "windows" { + volumeLen, dir = cleanGlobPathWindows(dir) + } else { + dir = cleanGlobPath(dir) + } + + if !hasMeta(dir[volumeLen:]) { + return glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, filepath.ErrBadPattern + } + + var m []string + m, err = Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(filepath.Separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +func volumeNameLen(path string) int { + isSlash := func(c uint8) bool { + return c == '\\' || c == '/' + } + if len(path) < 2 { + return 0 + } + // with drive letter + c := path[0] + if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return 2 + } + // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return n + } + break + } + } + } + return 0 +} + +// cleanGlobPathWindows is windows version of cleanGlobPath. +func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { + vollen := volumeNameLen(path) + switch { + case path == "": + return 0, "." + case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ + // do nothing to the path + return vollen + 1, path + case vollen == len(path) && len(path) == 2: // C: + return vollen, path + "." // convert C: into C:. + default: + if vollen >= len(path) { + vollen = len(path) - 1 + } + return vollen, path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := Stat(dir) + if err != nil { + return // ignore I/O error + } + if !fi.IsDir() { + return // ignore I/O error + } + + list, err := ReadDir(dir) + if err != nil { + return // ignore I/O error + } + + var names []string + for _, info := range list { + names = append(names, info.Name()) + } + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by filepath.Match. +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go new file mode 100644 index 0000000000..22ad2fe445 --- /dev/null +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -0,0 +1,1095 @@ +package fsys + +import ( + "cmd/go/internal/txtar" + "encoding/json" + "errors" + "fmt" + "internal/testenv" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +// initOverlay resets the overlay state to reflect the config. +// config should be a text archive string. The comment is the overlay config +// json, and the files, in the archive are laid out in a temp directory +// that cwd is set to. +func initOverlay(t *testing.T, config string) { + t.Helper() + + // Create a temporary directory and chdir to it. + prevwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + cwd = filepath.Join(t.TempDir(), "root") + if err := os.Mkdir(cwd, 0777); err != nil { + t.Fatal(err) + } + if err := os.Chdir(cwd); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + overlay = nil + if err := os.Chdir(prevwd); err != nil { + t.Fatal(err) + } + }) + + a := txtar.Parse([]byte(config)) + for _, f := range a.Files { + name := filepath.Join(cwd, f.Name) + if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(name, f.Data, 0666); err != nil { + t.Fatal(err) + } + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(a.Comment, &overlayJSON); err != nil { + t.Fatal(fmt.Errorf("parsing overlay JSON: %v", err)) + } + + initFromJSON(overlayJSON) +} + +func TestIsDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6": "" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- subdir6/file6.txt -- +six +`) + + testCases := []struct { + path string + want, wantErr bool + }{ + {"", true, true}, + {".", true, false}, + {cwd, true, false}, + {cwd + string(filepath.Separator), true, false}, + // subdir1 is only on disk + {filepath.Join(cwd, "subdir1"), true, false}, + {"subdir1", true, false}, + {"subdir1" + string(filepath.Separator), true, false}, + {"subdir1/file1.txt", false, false}, + {"subdir1/doesntexist.txt", false, true}, + {"doesntexist", false, true}, + // subdir2 is only in overlay + {filepath.Join(cwd, "subdir2"), true, false}, + {"subdir2", true, false}, + {"subdir2" + string(filepath.Separator), true, false}, + {"subdir2/file2.txt", false, false}, + {"subdir2/doesntexist.txt", false, true}, + // subdir3 has files on disk and in overlay + {filepath.Join(cwd, "subdir3"), true, false}, + {"subdir3", true, false}, + {"subdir3" + string(filepath.Separator), true, false}, + {"subdir3/file3a.txt", false, false}, + {"subdir3/file3b.txt", false, false}, + {"subdir3/doesntexist.txt", false, true}, + // subdir4 is overlaid with a file + {filepath.Join(cwd, "subdir4"), false, false}, + {"subdir4", false, false}, + {"subdir4" + string(filepath.Separator), false, false}, + {"subdir4/file4.txt", false, false}, + {"subdir4/doesntexist.txt", false, false}, + // subdir5 doesn't exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir5"), false, false}, + {"subdir5", false, false}, + {"subdir5" + string(filepath.Separator), false, false}, + {"subdir5/file5.txt", false, false}, + {"subdir5/doesntexist.txt", false, false}, + // subdir6 does exist, and is overlaid with a "delete" entry + {filepath.Join(cwd, "subdir6"), false, false}, + {"subdir6", false, false}, + {"subdir6" + string(filepath.Separator), false, false}, + {"subdir6/file6.txt", false, false}, + {"subdir6/doesntexist.txt", false, false}, + } + + for _, tc := range testCases { + got, err := IsDir(tc.path) + if err != nil { + if !tc.wantErr { + t.Errorf("IsDir(%q): got error with string %q, want no error", tc.path, err.Error()) + } + continue + } + if tc.wantErr { + t.Errorf("IsDir(%q): got no error, want error", tc.path) + } + if tc.want != got { + t.Errorf("IsDir(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +const readDirOverlay = ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir4": "overlayfiles/subdir4", + "subdir3/file3b.txt": "overlayfiles/subdir3_file3b.txt", + "subdir5": "", + "subdir6/asubsubdir/afile.txt": "overlayfiles/subdir6_asubsubdir_afile.txt", + "subdir6/asubsubdir/zfile.txt": "overlayfiles/subdir6_asubsubdir_zfile.txt", + "subdir6/zsubsubdir/file.txt": "overlayfiles/subdir6_zsubsubdir_file.txt", + "subdir7/asubsubdir/file.txt": "overlayfiles/subdir7_asubsubdir_file.txt", + "subdir7/zsubsubdir/file.txt": "overlayfiles/subdir7_zsubsubdir_file.txt", + "subdir8/doesntexist": "this_file_doesnt_exist_anywhere", + "other/pointstodir": "overlayfiles/this_is_a_directory", + "parentoverwritten/subdir1": "overlayfiles/parentoverwritten_subdir1", + "subdir9/this_file_is_overlaid.txt": "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir10/only_deleted_file.txt": "", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "textfile.txt/file.go": "overlayfiles/textfile_txt_file.go" + } +} +-- subdir1/file1.txt -- + +-- subdir3/file3a.txt -- +33 +-- subdir4/file4.txt -- +444 +-- subdir6/file.txt -- +-- subdir6/asubsubdir/file.txt -- +-- subdir6/anothersubsubdir/file.txt -- +-- subdir9/this_file_is_overlaid.txt -- +-- subdir10/only_deleted_file.txt -- +this will be deleted in overlay +-- subdir11/deleted.txt -- +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- textfile.txt -- +this will be overridden by textfile.txt/file.go +-- overlayfiles/subdir2_file2.txt -- +2 +-- overlayfiles/subdir3_file3b.txt -- +66666 +-- overlayfiles/subdir4 -- +x +-- overlayfiles/subdir6_asubsubdir_afile.txt -- +-- overlayfiles/subdir6_asubsubdir_zfile.txt -- +-- overlayfiles/subdir6_zsubsubdir_file.txt -- +-- overlayfiles/subdir7_asubsubdir_file.txt -- +-- overlayfiles/subdir7_zsubsubdir_file.txt -- +-- overlayfiles/parentoverwritten_subdir1 -- +x +-- overlayfiles/subdir9_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/subdir11 -- +-- overlayfiles/this_is_a_directory/file.txt -- +-- overlayfiles/textfile_txt_file.go -- +x +` + +func TestReadDir(t *testing.T) { + initOverlay(t, readDirOverlay) + + type entry struct { + name string + size int64 + isDir bool + } + + testCases := []struct { + dir string + want []entry + }{ + { + ".", []entry{ + {"other", 0, true}, + {"overlayfiles", 0, true}, + {"parentoverwritten", 0, true}, + {"subdir1", 0, true}, + {"subdir10", 0, true}, + {"subdir11", 0, false}, + {"subdir2", 0, true}, + {"subdir3", 0, true}, + {"subdir4", 2, false}, + // no subdir5. + {"subdir6", 0, true}, + {"subdir7", 0, true}, + {"subdir8", 0, true}, + {"subdir9", 0, true}, + {"textfile.txt", 0, true}, + }, + }, + { + "subdir1", []entry{ + {"file1.txt", 1, false}, + }, + }, + { + "subdir2", []entry{ + {"file2.txt", 2, false}, + }, + }, + { + "subdir3", []entry{ + {"file3a.txt", 3, false}, + {"file3b.txt", 6, false}, + }, + }, + { + "subdir6", []entry{ + {"anothersubsubdir", 0, true}, + {"asubsubdir", 0, true}, + {"file.txt", 0, false}, + {"zsubsubdir", 0, true}, + }, + }, + { + "subdir6/asubsubdir", []entry{ + {"afile.txt", 0, false}, + {"file.txt", 0, false}, + {"zfile.txt", 0, false}, + }, + }, + { + "subdir8", []entry{ + {"doesntexist", 0, false}, // entry is returned even if destination file doesn't exist + }, + }, + { + // check that read dir actually redirects files that already exist + // the original this_file_is_overlaid.txt is empty + "subdir9", []entry{ + {"this_file_is_overlaid.txt", 9, false}, + }, + }, + { + "subdir10", []entry{}, + }, + { + "parentoverwritten", []entry{ + {"subdir1", 2, false}, + }, + }, + { + "textfile.txt", []entry{ + {"file.go", 2, false}, + }, + }, + } + + for _, tc := range testCases { + dir, want := tc.dir, tc.want + infos, err := ReadDir(dir) + if err != nil { + t.Errorf("ReadDir(%q): %v", dir, err) + continue + } + // Sorted diff of want and infos. + for len(infos) > 0 || len(want) > 0 { + switch { + case len(want) == 0 || len(infos) > 0 && infos[0].Name() < want[0].name: + t.Errorf("ReadDir(%q): unexpected entry: %s IsDir=%v Size=%v", dir, infos[0].Name(), infos[0].IsDir(), infos[0].Size()) + infos = infos[1:] + case len(infos) == 0 || len(want) > 0 && want[0].name < infos[0].Name(): + t.Errorf("ReadDir(%q): missing entry: %s IsDir=%v Size=%v", dir, want[0].name, want[0].isDir, want[0].size) + want = want[1:] + default: + infoSize := infos[0].Size() + if want[0].isDir { + infoSize = 0 + } + if infos[0].IsDir() != want[0].isDir || want[0].isDir && infoSize != want[0].size { + t.Errorf("ReadDir(%q): %s: IsDir=%v Size=%v, want IsDir=%v Size=%v", dir, want[0].name, infos[0].IsDir(), infoSize, want[0].isDir, want[0].size) + } + infos = infos[1:] + want = want[1:] + } + } + } + + errCases := []string{ + "subdir1/file1.txt", // regular file on disk + "subdir2/file2.txt", // regular file in overlay + "subdir4", // directory overlaid with regular file + "subdir5", // directory deleted in overlay + "parentoverwritten/subdir1/subdir2/subdir3", // parentoverwritten/subdir1 overlaid with regular file + "parentoverwritten/subdir1/subdir2", // parentoverwritten/subdir1 overlaid with regular file + "subdir11", // directory with deleted child, overlaid with regular file + "other/pointstodir", + } + + for _, dir := range errCases { + _, err := ReadDir(dir) + if _, ok := err.(*fs.PathError); !ok { + t.Errorf("ReadDir(%q): err = %T (%v), want fs.PathError", dir, err, err) + } + } +} + +func TestGlob(t *testing.T) { + initOverlay(t, readDirOverlay) + + testCases := []struct { + pattern string + match []string + }{ + { + "*o*", + []string{ + "other", + "overlayfiles", + "parentoverwritten", + }, + }, + { + "subdir2/file2.txt", + []string{ + "subdir2/file2.txt", + }, + }, + { + "*/*.txt", + []string{ + "overlayfiles/subdir2_file2.txt", + "overlayfiles/subdir3_file3b.txt", + "overlayfiles/subdir6_asubsubdir_afile.txt", + "overlayfiles/subdir6_asubsubdir_zfile.txt", + "overlayfiles/subdir6_zsubsubdir_file.txt", + "overlayfiles/subdir7_asubsubdir_file.txt", + "overlayfiles/subdir7_zsubsubdir_file.txt", + "overlayfiles/subdir9_this_file_is_overlaid.txt", + "subdir1/file1.txt", + "subdir2/file2.txt", + "subdir3/file3a.txt", + "subdir3/file3b.txt", + "subdir6/file.txt", + "subdir9/this_file_is_overlaid.txt", + }, + }, + } + + for _, tc := range testCases { + pattern := tc.pattern + match, err := Glob(pattern) + if err != nil { + t.Errorf("Glob(%q): %v", pattern, err) + continue + } + want := tc.match + for i, name := range want { + if name != tc.pattern { + want[i] = filepath.FromSlash(name) + } + } + for len(match) > 0 || len(want) > 0 { + switch { + case len(match) == 0 || len(want) > 0 && want[0] < match[0]: + t.Errorf("Glob(%q): missing match: %s", pattern, want[0]) + want = want[1:] + case len(want) == 0 || len(match) > 0 && match[0] < want[0]: + t.Errorf("Glob(%q): extra match: %s", pattern, match[0]) + match = match[1:] + default: + want = want[1:] + match = match[1:] + } + } + } +} + +func TestOverlayPath(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "" + } +} +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- subdir5/deleted.txt -- +deleted +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +`) + + testCases := []struct { + path string + wantPath string + wantOK bool + }{ + {"subdir1/file1.txt", "subdir1/file1.txt", false}, + // OverlayPath returns false for directories + {"subdir2", "subdir2", false}, + {"subdir2/file2.txt", filepath.Join(cwd, "overlayfiles/subdir2_file2.txt"), true}, + // OverlayPath doesn't stat a file to see if it exists, so it happily returns + // the 'to' path and true even if the 'to' path doesn't exist on disk. + {"subdir3/doesntexist", filepath.Join(cwd, "this_file_doesnt_exist_anywhere"), true}, + // Like the subdir2/file2.txt case above, but subdir4 exists on disk, but subdir2 does not. + {"subdir4/this_file_is_overlaid.txt", filepath.Join(cwd, "overlayfiles/subdir4_this_file_is_overlaid.txt"), true}, + {"subdir5", "subdir5", false}, + {"subdir5/deleted.txt", "", true}, + } + + for _, tc := range testCases { + gotPath, gotOK := OverlayPath(tc.path) + if gotPath != tc.wantPath || gotOK != tc.wantOK { + t.Errorf("OverlayPath(%q): got %v, %v; want %v, %v", + tc.path, gotPath, gotOK, tc.wantPath, tc.wantOK) + } + } +} + +func TestOpen(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "subdir2/file2.txt": "overlayfiles/subdir2_file2.txt", + "subdir3/doesntexist": "this_file_doesnt_exist_anywhere", + "subdir4/this_file_is_overlaid.txt": "overlayfiles/subdir4_this_file_is_overlaid.txt", + "subdir5/deleted.txt": "", + "parentoverwritten/subdir1": "", + "childoverlay/subdir1.txt/child.txt": "overlayfiles/child.txt", + "subdir11/deleted.txt": "", + "subdir11": "overlayfiles/subdir11", + "parentdeleted": "", + "parentdeleted/file.txt": "overlayfiles/parentdeleted_file.txt" + } +} +-- subdir11/deleted.txt -- +-- subdir1/file1.txt -- +file 1 +-- subdir4/this_file_is_overlaid.txt -- +these contents are replaced by the overlay +-- parentoverwritten/subdir1/subdir2/subdir3/file.txt -- +-- childoverlay/subdir1.txt -- +this file doesn't exist because the path +childoverlay/subdir1.txt/child.txt is in the overlay +-- subdir5/deleted.txt -- +deleted +-- parentdeleted -- +this will be deleted so that parentdeleted/file.txt can exist +-- overlayfiles/subdir2_file2.txt -- +file 2 +-- overlayfiles/subdir4_this_file_is_overlaid.txt -- +99999999 +-- overlayfiles/child.txt -- +-- overlayfiles/subdir11 -- +11 +-- overlayfiles/parentdeleted_file.txt -- +this can exist because the parent directory is deleted +`) + + testCases := []struct { + path string + wantContents string + isErr bool + }{ + {"subdir1/file1.txt", "file 1\n", false}, + {"subdir2/file2.txt", "file 2\n", false}, + {"subdir3/doesntexist", "", true}, + {"subdir4/this_file_is_overlaid.txt", "99999999\n", false}, + {"subdir5/deleted.txt", "", true}, + {"parentoverwritten/subdir1/subdir2/subdir3/file.txt", "", true}, + {"childoverlay/subdir1.txt", "", true}, + {"subdir11", "11\n", false}, + {"parentdeleted/file.txt", "this can exist because the parent directory is deleted\n", false}, + } + + for _, tc := range testCases { + f, err := Open(tc.path) + if tc.isErr { + if err == nil { + f.Close() + t.Errorf("Open(%q): got no error, but want error", tc.path) + } + continue + } + if err != nil { + t.Errorf("Open(%q): got error %v, want nil", tc.path, err) + continue + } + contents, err := io.ReadAll(f) + if err != nil { + t.Errorf("unexpected error reading contents of file: %v", err) + } + if string(contents) != tc.wantContents { + t.Errorf("contents of file opened with Open(%q): got %q, want %q", + tc.path, contents, tc.wantContents) + } + f.Close() + } +} + +func TestIsDirWithGoFiles(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "goinoverlay/file.go": "dummy", + "directory/removed/by/file": "dummy", + "directory_with_go_dir/dir.go/file.txt": "dummy", + "otherdirectory/deleted.go": "", + "nonexistentdirectory/deleted.go": "", + "textfile.txt/file.go": "dummy" + } +} +-- dummy -- +a destination file for the overlay entries to point to +contents don't matter for this test +-- nogo/file.txt -- +-- goondisk/file.go -- +-- goinoverlay/file.txt -- +-- directory/removed/by/file/in/overlay/file.go -- +-- otherdirectory/deleted.go -- +-- textfile.txt -- +`) + + testCases := []struct { + dir string + want bool + wantErr bool + }{ + {"nogo", false, false}, + {"goondisk", true, false}, + {"goinoverlay", true, false}, + {"directory/removed/by/file/in/overlay", false, false}, + {"directory_with_go_dir", false, false}, + {"otherdirectory", false, false}, + {"nonexistentdirectory", false, false}, + {"textfile.txt", true, false}, + } + + for _, tc := range testCases { + got, gotErr := IsDirWithGoFiles(tc.dir) + if tc.wantErr { + if gotErr == nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want non-nil error", tc.dir, got, gotErr) + } + continue + } + if gotErr != nil { + t.Errorf("IsDirWithGoFiles(%q): got %v, %v; want nil error", tc.dir, got, gotErr) + } + if got != tc.want { + t.Errorf("IsDirWithGoFiles(%q) = %v; want %v", tc.dir, got, tc.want) + } + } +} + +func TestWalk(t *testing.T) { + // The root of the walk must be a name with an actual basename, not just ".". + // Walk uses Lstat to obtain the name of the root, and Lstat on platforms + // other than Plan 9 reports the name "." instead of the actual base name of + // the directory. (See https://golang.org/issue/42115.) + + type file struct { + path string + name string + size int64 + mode fs.FileMode + isDir bool + } + testCases := []struct { + name string + overlay string + root string + wantFiles []file + }{ + {"no overlay", ` +{} +-- dir/file.txt -- +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0700, true}, + {"dir/file.txt", "file.txt", 0, 0600, false}, + }, + }, + {"overlay with different file", ` +{ + "Replace": { + "dir/file.txt": "dir/other.txt" + } +} +-- dir/file.txt -- +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new file", ` +{ + "Replace": { + "dir/file.txt": "dir/other.txt" + } +} +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/file.txt", "file.txt", 23, 0600, false}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + }, + }, + {"overlay with new directory", ` +{ + "Replace": { + "dir/subdir/file.txt": "dir/other.txt" + } +} +-- dir/other.txt -- +contents of other file +`, + "dir", + []file{ + {"dir", "dir", 0, fs.ModeDir | 0500, true}, + {"dir/other.txt", "other.txt", 23, 0600, false}, + {"dir/subdir", "subdir", 0, fs.ModeDir | 0500, true}, + {"dir/subdir/file.txt", "file.txt", 23, 0600, false}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + + var got []file + Walk(tc.root, func(path string, info fs.FileInfo, err error) error { + got = append(got, file{path, info.Name(), info.Size(), info.Mode(), info.IsDir()}) + return nil + }) + + if len(got) != len(tc.wantFiles) { + t.Errorf("Walk: saw %#v in walk; want %#v", got, tc.wantFiles) + } + for i := 0; i < len(got) && i < len(tc.wantFiles); i++ { + wantPath := filepath.FromSlash(tc.wantFiles[i].path) + if got[i].path != wantPath { + t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, wantPath) + } + if got[i].name != tc.wantFiles[i].name { + t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name) + } + if got[i].mode&(fs.ModeDir|0700) != tc.wantFiles[i].mode { + t.Errorf("mode&(fs.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(fs.ModeDir|0700), tc.wantFiles[i].mode) + } + if got[i].isDir != tc.wantFiles[i].isDir { + t.Errorf("isDir for file #%v in walk, got %v, want %v", i, got[i].isDir, tc.wantFiles[i].isDir) + } + if tc.wantFiles[i].isDir { + continue // don't check size for directories + } + if got[i].size != tc.wantFiles[i].size { + t.Errorf("size of file #%v in walk, got %v, want %v", i, got[i].size, tc.wantFiles[i].size) + } + } + }) + } +} + +func TestWalkSkipDir(t *testing.T) { + initOverlay(t, ` +{ + "Replace": { + "dir/skip/file.go": "dummy.txt", + "dir/dontskip/file.go": "dummy.txt", + "dir/dontskip/skip/file.go": "dummy.txt" + } +} +-- dummy.txt -- +`) + + var seen []string + Walk("dir", func(path string, info fs.FileInfo, err error) error { + seen = append(seen, filepath.ToSlash(path)) + if info.Name() == "skip" { + return filepath.SkipDir + } + return nil + }) + + wantSeen := []string{"dir", "dir/dontskip", "dir/dontskip/file.go", "dir/dontskip/skip", "dir/skip"} + + if len(seen) != len(wantSeen) { + t.Errorf("paths seen in walk: got %v entries; want %v entries", len(seen), len(wantSeen)) + } + + for i := 0; i < len(seen) && i < len(wantSeen); i++ { + if seen[i] != wantSeen[i] { + t.Errorf("path #%v seen walking tree: want %q, got %q", i, seen[i], wantSeen[i]) + } + } +} + +func TestWalkError(t *testing.T) { + initOverlay(t, "{}") + + alreadyCalled := false + err := Walk("foo", func(path string, info fs.FileInfo, err error) error { + if alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was called more than once") + } + alreadyCalled = true + return errors.New("returned from function") + }) + if !alreadyCalled { + t.Fatal("expected walk function to be called exactly once, but it was never called") + + } + if err == nil { + t.Fatalf("Walk: got no error, want error") + } + if err.Error() != "returned from function" { + t.Fatalf("Walk: got error %v, want \"returned from function\" error", err) + } +} + +func TestWalkSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + initOverlay(t, `{ + "Replace": {"overlay_symlink": "symlink"} +} +-- dir/file --`) + + // Create symlink + if err := os.Symlink("dir", "symlink"); err != nil { + t.Error(err) + } + + testCases := []struct { + name string + dir string + wantFiles []string + }{ + {"control", "dir", []string{"dir", "dir" + string(filepath.Separator) + "file"}}, + // ensure Walk doesn't walk into the directory pointed to by the symlink + // (because it's supposed to use Lstat instead of Stat). + {"symlink_to_dir", "symlink", []string{"symlink"}}, + {"overlay_to_symlink_to_dir", "overlay_symlink", []string{"overlay_symlink"}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var got []string + + err := Walk(tc.dir, func(path string, info fs.FileInfo, err error) error { + got = append(got, path) + if err != nil { + t.Errorf("walkfn: got non nil err argument: %v, want nil err argument", err) + } + return nil + }) + if err != nil { + t.Errorf("Walk: got error %q, want nil", err) + } + + if !reflect.DeepEqual(got, tc.wantFiles) { + t.Errorf("files examined by walk: got %v, want %v", got, tc.wantFiles) + } + }) + } + +} + +func TestLstat(t *testing.T) { + type file struct { + name string + size int64 + mode fs.FileMode // mode & (fs.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | fs.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | fs.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := lstat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("lstat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("lstat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("lstat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(fs.ModeDir|0700) != tc.want.mode { + t.Errorf("lstat(%q).Mode()&(fs.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(fs.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("lstat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("lstat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} + +func TestStat(t *testing.T) { + testenv.MustHaveSymlink(t) + + type file struct { + name string + size int64 + mode os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions + isDir bool + } + + testCases := []struct { + name string + overlay string + path string + + want file + wantErr bool + }{ + { + "regular_file", + `{} +-- file.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "new_file_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_replaced_in_overlay", + `{"Replace": {"file.txt": "dummy.txt"}} +-- file.txt -- +-- dummy.txt -- +contents`, + "file.txt", + file{"file.txt", 9, 0600, false}, + false, + }, + { + "file_cant_exist", + `{"Replace": {"deleted": "dummy.txt"}} +-- deleted/file.txt -- +-- dummy.txt -- +`, + "deleted/file.txt", + file{}, + true, + }, + { + "deleted", + `{"Replace": {"deleted": ""}} +-- deleted -- +`, + "deleted", + file{}, + true, + }, + { + "dir_on_disk", + `{} +-- dir/foo.txt -- +`, + "dir", + file{"dir", 0, 0700 | os.ModeDir, true}, + false, + }, + { + "dir_in_overlay", + `{"Replace": {"dir/file.txt": "dummy.txt"}} +-- dummy.txt -- +`, + "dir", + file{"dir", 0, 0500 | os.ModeDir, true}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + initOverlay(t, tc.overlay) + got, err := Stat(tc.path) + if tc.wantErr { + if err == nil { + t.Errorf("Stat(%q): got no error, want error", tc.path) + } + return + } + if err != nil { + t.Fatalf("Stat(%q): got error %v, want no error", tc.path, err) + } + if got.Name() != tc.want.name { + t.Errorf("Stat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name) + } + if got.Mode()&(os.ModeDir|0700) != tc.want.mode { + t.Errorf("Stat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode) + } + if got.IsDir() != tc.want.isDir { + t.Errorf("Stat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir) + } + if tc.want.isDir { + return // don't check size for directories + } + if got.Size() != tc.want.size { + t.Errorf("Stat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size) + } + }) + } +} + +func TestStatSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + initOverlay(t, `{ + "Replace": {"file.go": "symlink"} +} +-- to.go -- +0123456789 +`) + + // Create symlink + if err := os.Symlink("to.go", "symlink"); err != nil { + t.Error(err) + } + + f := "file.go" + fi, err := Stat(f) + if err != nil { + t.Errorf("Stat(%q): got error %q, want nil error", f, err) + } + + if !fi.Mode().IsRegular() { + t.Errorf("Stat(%q).Mode(): got %v, want regular mode", f, fi.Mode()) + } + + if fi.Size() != 11 { + t.Errorf("Stat(%q).Size(): got %v, want 11", f, fi.Size()) + } +} diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ed2786879c..268962eca8 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -46,8 +46,8 @@ before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving custom domains using insecure schemes such as HTTP. Use with caution. This flag is deprecated and will be removed in a future version of go. -The GOINSECURE environment variable is usually a better alternative, since -it provides control over which modules may be retrieved using an insecure +The GOINSECURE environment variable should be used instead, since it +provides control over which packages may be retrieved using an insecure scheme. See 'go help environment' for details. The -t flag instructs get to also download the packages required to build diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 0ae5fd7ca9..8dfabbaa4a 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -582,8 +582,8 @@ Architecture-specific environment variables: For GOARCH=arm, the ARM architecture for which to compile. Valid values are 5, 6, 7. GO386 - For GOARCH=386, the floating point instruction set. - Valid values are 387, sse2. + For GOARCH=386, how to implement floating point instructions. + Valid values are sse2 (default), softfloat. GOMIPS For GOARCH=mips{,le}, whether to use floating point instructions. Valid values are hardfloat (default), softfloat. diff --git a/src/cmd/go/internal/imports/read.go b/src/cmd/go/internal/imports/read.go index 58c2abdc29..5e270781d7 100644 --- a/src/cmd/go/internal/imports/read.go +++ b/src/cmd/go/internal/imports/read.go @@ -198,7 +198,7 @@ func (r *importReader) readImport(imports *[]string) { r.readString(imports) } -// ReadComments is like ioutil.ReadAll, except that it only reads the leading +// ReadComments is like io.ReadAll, except that it only reads the leading // block of comments in the file. func ReadComments(f io.Reader) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} @@ -210,7 +210,7 @@ func ReadComments(f io.Reader) ([]byte, error) { return r.buf, r.err } -// ReadImports is like ioutil.ReadAll, except that it expects a Go file as input +// ReadImports is like io.ReadAll, except that it expects a Go file as input // and stops reading the input once the imports have completed. func ReadImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, error) { r := &importReader{b: bufio.NewReader(f)} diff --git a/src/cmd/go/internal/imports/scan.go b/src/cmd/go/internal/imports/scan.go index 3d9b6132b1..ee11a8708b 100644 --- a/src/cmd/go/internal/imports/scan.go +++ b/src/cmd/go/internal/imports/scan.go @@ -6,16 +6,17 @@ package imports import ( "fmt" - "io/ioutil" - "os" + "io/fs" "path/filepath" "sort" "strconv" "strings" + + "cmd/go/internal/fsys" ) func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { - infos, err := ioutil.ReadDir(dir) + infos, err := fsys.ReadDir(dir) if err != nil { return nil, nil, err } @@ -25,14 +26,14 @@ func ScanDir(dir string, tags map[string]bool) ([]string, []string, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. - if info.Mode()&os.ModeSymlink != 0 { - info, err = os.Stat(filepath.Join(dir, name)) + if info.Mode()&fs.ModeSymlink != 0 { + info, err = fsys.Stat(filepath.Join(dir, name)) if err != nil { continue // Ignore broken symlinks. } } - if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { + if info.Mode().IsRegular() && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && MatchFile(name, tags) { files = append(files, filepath.Join(dir, name)) } } @@ -49,7 +50,7 @@ func scanFiles(files []string, tags map[string]bool, explicitFiles bool) ([]stri numFiles := 0 Files: for _, name := range files { - r, err := os.Open(name) + r, err := fsys.Open(name) if err != nil { return nil, nil, err } diff --git a/src/cmd/go/internal/imports/tags.go b/src/cmd/go/internal/imports/tags.go index 14b4e21a02..01b448b914 100644 --- a/src/cmd/go/internal/imports/tags.go +++ b/src/cmd/go/internal/imports/tags.go @@ -4,17 +4,23 @@ package imports -import "cmd/go/internal/cfg" +import ( + "cmd/go/internal/cfg" + "sync" +) -var tags map[string]bool +var ( + tags map[string]bool + tagsOnce sync.Once +) // Tags returns a set of build tags that are true for the target platform. // It includes GOOS, GOARCH, the compiler, possibly "cgo", // release tags like "go1.13", and user-specified build tags. func Tags() map[string]bool { - if tags == nil { + tagsOnce.Do(func() { tags = loadTags() - } + }) return tags } @@ -36,14 +42,17 @@ func loadTags() map[string]bool { return tags } -var anyTags map[string]bool +var ( + anyTags map[string]bool + anyTagsOnce sync.Once +) // AnyTags returns a special set of build tags that satisfy nearly all // build tag expressions. Only "ignore" and malformed build tag requirements // are considered false. func AnyTags() map[string]bool { - if anyTags == nil { + anyTagsOnce.Do(func() { anyTags = map[string]bool{"*": true} - } + }) return anyTags } diff --git a/src/cmd/go/internal/imports/testdata/android/.h.go b/src/cmd/go/internal/imports/testdata/android/.h.go new file mode 100644 index 0000000000..53c529e777 --- /dev/null +++ b/src/cmd/go/internal/imports/testdata/android/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" diff --git a/src/cmd/go/internal/imports/testdata/illumos/.h.go b/src/cmd/go/internal/imports/testdata/illumos/.h.go new file mode 100644 index 0000000000..53c529e777 --- /dev/null +++ b/src/cmd/go/internal/imports/testdata/illumos/.h.go @@ -0,0 +1,3 @@ +package android + +import _ "h" diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 33409eb774..89088f5def 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -66,26 +66,28 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is: BinaryOnly bool // binary-only package (no longer supported) ForTest string // package is only for use in named test Export string // file containing export data (when using -export) + BuildID string // build ID of the compiled package (when using -export) Module *Module // info about package's containing module, if any (can be nil) Match []string // command-line patterns matching this package DepOnly bool // package is only a dependency, not explicitly listed // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go source files that import "C" - CompiledGoFiles []string // .go files presented to compiler (when using -compiled) - IgnoredGoFiles []string // .go source files ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - FFiles []string // .f, .F, .for and .f90 Fortran source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - TestGoFiles []string // _test.go files in package - XTestGoFiles []string // _test.go files outside package + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + CompiledGoFiles []string // .go files presented to compiler (when using -compiled) + IgnoredGoFiles []string // .go source files ignored due to build constraints + IgnoredOtherFiles []string // non-.go source files ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package // Cgo directives CgoCFLAGS []string // cgo: flags for C compiler @@ -413,7 +415,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. + modload.LoadModFile(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 5cc77915e7..4c541b9017 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -14,6 +14,7 @@ import ( "go/build" "go/scanner" "go/token" + "io/fs" "io/ioutil" "os" pathpkg "path" @@ -27,12 +28,14 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/sys" ) var IgnoreImports bool // control whether we ignore imports in packages @@ -58,6 +61,7 @@ type PackagePublic struct { ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory ForTest string `json:",omitempty"` // package is only for use in named test Export string `json:",omitempty"` // file containing export data (set by go list -export) + BuildID string `json:",omitempty"` // build ID of the compiled package (set by go list -export) Module *modinfo.ModulePublic `json:",omitempty"` // info about package's module, if any Match []string `json:",omitempty"` // command-line patterns matching this package Goroot bool `json:",omitempty"` // is this package found in the Go root? @@ -75,19 +79,20 @@ type PackagePublic struct { // Source files // If you add to this list you MUST add to p.AllFiles (below) too. // Otherwise file name security lists will not apply to any new additions. - GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string `json:",omitempty"` // .go source files that import "C" - CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles - IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints - CFiles []string `json:",omitempty"` // .c source files - CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files - MFiles []string `json:",omitempty"` // .m source files - HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files - FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files - SFiles []string `json:",omitempty"` // .s source files - SwigFiles []string `json:",omitempty"` // .swig files - SwigCXXFiles []string `json:",omitempty"` // .swigcxx files - SysoFiles []string `json:",omitempty"` // .syso system object files added to package + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go source files that import "C" + CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles + IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints + IgnoredOtherFiles []string `json:",omitempty"` // non-.go source files ignored due to build constraints + CFiles []string `json:",omitempty"` // .c source files + CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files + MFiles []string `json:",omitempty"` // .m source files + HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files + FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files + SFiles []string `json:",omitempty"` // .s source files + SwigFiles []string `json:",omitempty"` // .swig files + SwigCXXFiles []string `json:",omitempty"` // .swigcxx files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package // Cgo directives CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler @@ -127,6 +132,7 @@ func (p *Package) AllFiles() []string { p.CgoFiles, // no p.CompiledGoFiles, because they are from GoFiles or generated by us p.IgnoredGoFiles, + p.IgnoredOtherFiles, p.CFiles, p.CXXFiles, p.MFiles, @@ -185,7 +191,7 @@ type NoGoError struct { } func (e *NoGoError) Error() string { - if len(e.Package.constraintIgnoredGoFiles()) > 0 { + if len(e.Package.IgnoredGoFiles) > 0 { // Go files exist, but they were ignored due to build constraints. return "build constraints exclude all Go files in " + e.Package.Dir } @@ -248,8 +254,8 @@ func (p *Package) setLoadPackageDataError(err error, path string, stk *ImportSta // package's source files themselves (scanner errors). // // TODO(matloob): Perhaps make each of those the errors in the first group - // (including modload.ImportMissingError, and the corresponding - // "cannot find package %q in any of" GOPATH-mode error + // (including modload.ImportMissingError, ImportMissingSumError, and the + // corresponding "cannot find package %q in any of" GOPATH-mode error // produced in build.(*Context).Import; modload.AmbiguousImportError, // and modload.PackageNotInModuleError; and the malformed module path errors // produced in golang.org/x/mod/module.CheckMod) implement an interface @@ -330,6 +336,7 @@ func (p *Package) copyBuild(pp *build.Package) { p.GoFiles = pp.GoFiles p.CgoFiles = pp.CgoFiles p.IgnoredGoFiles = pp.IgnoredGoFiles + p.IgnoredOtherFiles = pp.IgnoredOtherFiles p.CFiles = pp.CFiles p.CXXFiles = pp.CXXFiles p.MFiles = pp.MFiles @@ -423,6 +430,7 @@ type ImportPathError interface { var ( _ ImportPathError = (*importError)(nil) _ ImportPathError = (*modload.ImportMissingError)(nil) + _ ImportPathError = (*modload.ImportMissingSumError)(nil) ) type importError struct { @@ -971,7 +979,7 @@ var isDirCache par.Cache func isDir(path string) bool { return isDirCache.Do(path, func() interface{} { - fi, err := os.Stat(path) + fi, err := fsys.Stat(path) return err == nil && fi.IsDir() }).(bool) } @@ -1944,37 +1952,44 @@ func LinkerDeps(p *Package) []string { // externalLinkingForced reports whether external linking is being // forced even for programs that do not use cgo. func externalLinkingForced(p *Package) bool { + if !cfg.BuildContext.CgoEnabled { + return false + } + // Some targets must use external linking even inside GOROOT. switch cfg.BuildContext.GOOS { case "android": if cfg.BuildContext.GOARCH != "arm64" { return true } - case "darwin", "ios": + case "ios": + return true + case "darwin": if cfg.BuildContext.GOARCH == "arm64" { return true } } - if !cfg.BuildContext.CgoEnabled { - return false - } // Currently build modes c-shared, pie (on systems that do not // support PIE with internal linking mode (currently all // systems: issue #18968)), plugin, and -linkshared force // external linking mode, as of course does // -ldflags=-linkmode=external. External linking mode forces // an import of runtime/cgo. - pieCgo := cfg.BuildBuildmode == "pie" + // If there are multiple -linkmode options, the last one wins. + pieCgo := cfg.BuildBuildmode == "pie" && !sys.InternalLinkPIESupported(cfg.BuildContext.GOOS, cfg.BuildContext.GOARCH) linkmodeExternal := false if p != nil { ldflags := BuildLdflags.For(p) - for i, a := range ldflags { - if a == "-linkmode=external" { - linkmodeExternal = true - } - if a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { + for i := len(ldflags) - 1; i >= 0; i-- { + a := ldflags[i] + if a == "-linkmode=external" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "external" { linkmodeExternal = true + break + } else if a == "-linkmode=internal" || + a == "-linkmode" && i+1 < len(ldflags) && ldflags[i+1] == "internal" { + break } } } @@ -2009,22 +2024,7 @@ func (p *Package) InternalXGoFiles() []string { // using absolute paths. "Possibly relevant" means that files are not excluded // due to build tags, but files with names beginning with . or _ are still excluded. func (p *Package) InternalAllGoFiles() []string { - return p.mkAbs(str.StringList(p.constraintIgnoredGoFiles(), p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) -} - -// constraintIgnoredGoFiles returns the list of Go files ignored for reasons -// other than having a name beginning with '.' or '_'. -func (p *Package) constraintIgnoredGoFiles() []string { - if len(p.IgnoredGoFiles) == 0 { - return nil - } - files := make([]string, 0, len(p.IgnoredGoFiles)) - for _, f := range p.IgnoredGoFiles { - if f != "" && f[0] != '.' && f[0] != '_' { - files = append(files, f) - } - } - return files + return p.mkAbs(str.StringList(p.IgnoredGoFiles, p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)) } // usesSwig reports whether the package needs to run SWIG. @@ -2153,7 +2153,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if strings.HasSuffix(p, ".go") { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). - if fi, err := os.Stat(p); err == nil && !fi.IsDir() { + if fi, err := fsys.Stat(p); err == nil && !fi.IsDir() { return []*Package{GoFilesPackage(ctx, patterns)} } } @@ -2310,10 +2310,10 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { // to make it look like this is a standard package or // command directory. So that local imports resolve // consistently, the files must all be in the same directory. - var dirent []os.FileInfo + var dirent []fs.FileInfo var dir string for _, file := range gofiles { - fi, err := os.Stat(file) + fi, err := fsys.Stat(file) if err != nil { base.Fatalf("%s", err) } @@ -2331,7 +2331,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { } dirent = append(dirent, fi) } - ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } + ctxt.ReadDir = func(string) ([]fs.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { modload.ImportFromFiles(ctx, gofiles) diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go index aba3eed776..05f27c321a 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock.go @@ -9,6 +9,7 @@ package filelock import ( "errors" + "io/fs" "os" ) @@ -24,7 +25,7 @@ type File interface { Fd() uintptr // Stat returns the FileInfo structure describing file. - Stat() (os.FileInfo, error) + Stat() (fs.FileInfo, error) } // Lock places an advisory write lock on the file, blocking until it can be @@ -87,7 +88,7 @@ var ErrNotSupported = errors.New("operation not supported") // underlyingError returns the underlying error for known os error types. func underlyingError(err error) error { switch err := err.(type) { - case *os.PathError: + case *fs.PathError: return err.Err case *os.LinkError: return err.Err diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go index 8776c5741c..1fa4327a89 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -18,8 +18,8 @@ package filelock import ( "errors" "io" + "io/fs" "math/rand" - "os" "sync" "syscall" "time" @@ -61,7 +61,7 @@ func lock(f File, lt lockType) (err error) { mu.Lock() if i, dup := inodes[f]; dup && i != ino { mu.Unlock() - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: errors.New("inode for file changed since last Lock or RLock"), @@ -152,7 +152,7 @@ func lock(f File, lt lockType) (err error) { if err != nil { unlock(f) - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go index 107611e1ce..bc480343fc 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go @@ -6,7 +6,7 @@ package filelock -import "os" +import "io/fs" type lockType int8 @@ -16,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -24,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go index afdffe323f..0798ee469a 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go @@ -6,9 +6,7 @@ package filelock -import ( - "os" -) +import "io/fs" type lockType int8 @@ -18,7 +16,7 @@ const ( ) func lock(f File, lt lockType) error { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: ErrNotSupported, @@ -26,7 +24,7 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: ErrNotSupported, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go index 78f2c51129..ed07bac608 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go @@ -7,7 +7,7 @@ package filelock import ( - "os" + "io/fs" "syscall" ) @@ -26,7 +26,7 @@ func lock(f File, lt lockType) (err error) { } } if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go index 43e85e450e..19de27eb9b 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go @@ -8,7 +8,7 @@ package filelock import ( "internal/syscall/windows" - "os" + "io/fs" "syscall" ) @@ -34,7 +34,7 @@ func lock(f File, lt lockType) error { err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: lt.String(), Path: f.Name(), Err: err, @@ -47,7 +47,7 @@ func unlock(f File) error { ol := new(syscall.Overlapped) err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) if err != nil { - return &os.PathError{ + return &fs.PathError{ Op: "Unlock", Path: f.Name(), Err: err, diff --git a/src/cmd/go/internal/lockedfile/lockedfile.go b/src/cmd/go/internal/lockedfile/lockedfile.go index 59b2dba44c..82e1a89675 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile.go +++ b/src/cmd/go/internal/lockedfile/lockedfile.go @@ -9,7 +9,7 @@ package lockedfile import ( "fmt" "io" - "io/ioutil" + "io/fs" "os" "runtime" ) @@ -35,7 +35,7 @@ type osFile struct { // OpenFile is like os.OpenFile, but returns a locked file. // If flag includes os.O_WRONLY or os.O_RDWR, the file is write-locked; // otherwise, it is read-locked. -func OpenFile(name string, flag int, perm os.FileMode) (*File, error) { +func OpenFile(name string, flag int, perm fs.FileMode) (*File, error) { var ( f = new(File) err error @@ -82,10 +82,10 @@ func Edit(name string) (*File, error) { // non-nil error. func (f *File) Close() error { if f.closed { - return &os.PathError{ + return &fs.PathError{ Op: "close", Path: f.Name(), - Err: os.ErrClosed, + Err: fs.ErrClosed, } } f.closed = true @@ -103,12 +103,12 @@ func Read(name string) ([]byte, error) { } defer f.Close() - return ioutil.ReadAll(f) + return io.ReadAll(f) } // Write opens the named file (creating it with the given permissions if needed), // then write-locks it and overwrites it with the given content. -func Write(name string, content io.Reader, perm os.FileMode) (err error) { +func Write(name string, content io.Reader, perm fs.FileMode) (err error) { f, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err @@ -135,7 +135,7 @@ func Transform(name string, t func([]byte) ([]byte, error)) (err error) { } defer f.Close() - old, err := ioutil.ReadAll(f) + old, err := io.ReadAll(f) if err != nil { return err } diff --git a/src/cmd/go/internal/lockedfile/lockedfile_filelock.go b/src/cmd/go/internal/lockedfile/lockedfile_filelock.go index f63dd8664b..10e1240efd 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile_filelock.go +++ b/src/cmd/go/internal/lockedfile/lockedfile_filelock.go @@ -7,12 +7,13 @@ package lockedfile import ( + "io/fs" "os" "cmd/go/internal/lockedfile/internal/filelock" ) -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // On BSD systems, we could add the O_SHLOCK or O_EXLOCK flag to the OpenFile // call instead of locking separately, but we have to support separate locking // calls for Linux and Windows anyway, so it's simpler to use that approach diff --git a/src/cmd/go/internal/lockedfile/lockedfile_plan9.go b/src/cmd/go/internal/lockedfile/lockedfile_plan9.go index 4a52c94976..51681381d7 100644 --- a/src/cmd/go/internal/lockedfile/lockedfile_plan9.go +++ b/src/cmd/go/internal/lockedfile/lockedfile_plan9.go @@ -7,6 +7,7 @@ package lockedfile import ( + "io/fs" "math/rand" "os" "strings" @@ -41,7 +42,7 @@ func isLocked(err error) bool { return false } -func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { // Plan 9 uses a mode bit instead of explicit lock/unlock syscalls. // // Per http://man.cat-v.org/plan_9/5/stat: “Exclusive use files may be open @@ -56,8 +57,8 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { // have the ModeExclusive bit set. Set it before we call OpenFile, so that we // can be confident that a successful OpenFile implies exclusive use. if fi, err := os.Stat(name); err == nil { - if fi.Mode()&os.ModeExclusive == 0 { - if err := os.Chmod(name, fi.Mode()|os.ModeExclusive); err != nil { + if fi.Mode()&fs.ModeExclusive == 0 { + if err := os.Chmod(name, fi.Mode()|fs.ModeExclusive); err != nil { return nil, err } } @@ -68,7 +69,7 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { nextSleep := 1 * time.Millisecond const maxSleep = 500 * time.Millisecond for { - f, err := os.OpenFile(name, flag, perm|os.ModeExclusive) + f, err := os.OpenFile(name, flag, perm|fs.ModeExclusive) if err == nil { return f, nil } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 050a2e0e12..e2e8ba6825 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -87,7 +87,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod(ctx) // to fill Target + modload.LoadModFile(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 7cfc0e6f5b..7384f3f293 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -10,8 +10,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" "context" - "os" - "strings" ) var cmdInit = &base.Command{ @@ -33,21 +31,14 @@ func init() { } func runInit(ctx context.Context, cmd *base.Command, args []string) { - modload.CmdModInit = true if len(args) > 1 { base.Fatalf("go mod init: too many arguments") } + var modPath string if len(args) == 1 { - modload.CmdModModule = args[0] + modPath = args[0] } + modload.ForceUseModules = true - modFilePath := modload.ModFilePath() - if _, err := os.Stat(modFilePath); err == nil { - base.Fatalf("go mod init: go.mod already exists") - } - if strings.Contains(modload.CmdModModule, "@") { - base.Fatalf("go mod init: module path must not contain '@'") - } - modload.InitMod(ctx) // does all the hard work - modload.WriteGoMod() + modload.CreateModFile(ctx, modPath) // does all the hard work } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 1bc4ab3def..1b9ce60529 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -232,7 +233,7 @@ var metaPrefixes = []string{ } // matchMetadata reports whether info is a metadata file. -func matchMetadata(dir string, info os.FileInfo) bool { +func matchMetadata(dir string, info fs.FileInfo) bool { name := info.Name() for _, p := range metaPrefixes { if strings.HasPrefix(name, p) { @@ -243,7 +244,7 @@ func matchMetadata(dir string, info os.FileInfo) bool { } // matchPotentialSourceFile reports whether info may be relevant to a build operation. -func matchPotentialSourceFile(dir string, info os.FileInfo) bool { +func matchPotentialSourceFile(dir string, info fs.FileInfo) bool { if strings.HasSuffix(info.Name(), "_test.go") { return false } @@ -269,7 +270,7 @@ func matchPotentialSourceFile(dir string, info os.FileInfo) bool { } // copyDir copies all regular files satisfying match(info) from src to dst. -func copyDir(dst, src string, match func(dir string, info os.FileInfo) bool) { +func copyDir(dst, src string, match func(dir string, info fs.FileInfo) bool) { files, err := ioutil.ReadDir(src) if err != nil { base.Fatalf("go mod vendor: %v", err) diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index bd591d3f32..ce24793929 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "io/fs" "io/ioutil" "os" "runtime" @@ -88,8 +89,8 @@ func verifyMod(mod module.Version) []error { dir, dirErr := modfetch.DownloadDir(mod) data, err := ioutil.ReadFile(zip + "hash") if err != nil { - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) && - dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) && + dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // Nothing downloaded yet. Nothing to verify. return nil } @@ -98,7 +99,7 @@ func verifyMod(mod module.Version) []error { } h := string(bytes.TrimSpace(data)) - if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) { + if zipErr != nil && errors.Is(zipErr, fs.ErrNotExist) { // ok } else { hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) @@ -109,7 +110,7 @@ func verifyMod(mod module.Version) []error { errs = append(errs, fmt.Errorf("%s %s: zip has been modified (%v)", mod.Path, mod.Version, zip)) } } - if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { + if dirErr != nil && errors.Is(dirErr, fs.ErrNotExist) { // ok } else { hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go index e3074b775e..b7aa670250 100644 --- a/src/cmd/go/internal/modfetch/cache.go +++ b/src/cmd/go/internal/modfetch/cache.go @@ -10,10 +10,12 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -59,7 +61,7 @@ func CachePath(m module.Version, suffix string) (string, error) { // DownloadDir returns the directory to which m should have been downloaded. // An error will be returned if the module path or version cannot be escaped. -// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned // along with the directory if the directory does not exist or if the directory // is not completely populated. func DownloadDir(m module.Version) (string, error) { @@ -106,14 +108,14 @@ func DownloadDir(m module.Version) (string, error) { // DownloadDirPartialError is returned by DownloadDir if a module directory // exists but was not completely populated. // -// DownloadDirPartialError is equivalent to os.ErrNotExist. +// DownloadDirPartialError is equivalent to fs.ErrNotExist. type DownloadDirPartialError struct { Dir string Err error } func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } -func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } +func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist } // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. @@ -155,16 +157,30 @@ func SideLock() (unlock func(), err error) { type cachingRepo struct { path string cache par.Cache // cache for all operations - r Repo + + once sync.Once + initRepo func() (Repo, error) + r Repo } -func newCachingRepo(r Repo) *cachingRepo { +func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo { return &cachingRepo{ - r: r, - path: r.ModulePath(), + path: path, + initRepo: initRepo, } } +func (r *cachingRepo) repo() Repo { + r.once.Do(func() { + var err error + r.r, err = r.initRepo() + if err != nil { + r.r = errRepo{r.path, err} + } + }) + return r.r +} + func (r *cachingRepo) ModulePath() string { return r.path } @@ -175,7 +191,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { err error } c := r.cache.Do("versions:"+prefix, func() interface{} { - list, err := r.r.Versions(prefix) + list, err := r.repo().Versions(prefix) return cached{list, err} }).(cached) @@ -197,7 +213,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { return cachedInfo{info, nil} } - info, err = r.r.Stat(rev) + info, err = r.repo().Stat(rev) if err == nil { // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, // then save the information under the proper version, for future use. @@ -224,7 +240,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { func (r *cachingRepo) Latest() (*RevInfo, error) { c := r.cache.Do("latest:", func() interface{} { - info, err := r.r.Latest() + info, err := r.repo().Latest() // Save info for likely future Stat call. if err == nil { @@ -258,7 +274,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { return cached{text, nil} } - text, err = r.r.GoMod(version) + text, err = r.repo().GoMod(version) if err == nil { if err := checkGoMod(r.path, version, text); err != nil { return cached{text, err} @@ -277,26 +293,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { } func (r *cachingRepo) Zip(dst io.Writer, version string) error { - return r.r.Zip(dst, version) -} - -// Stat is like Lookup(path).Stat(rev) but avoids the -// repository path resolution in Lookup if the result is -// already cached on local disk. -func Stat(proxy, path, rev string) (*RevInfo, error) { - _, info, err := readDiskStat(path, rev) - if err == nil { - return info, nil - } - repo, err := Lookup(proxy, path) - if err != nil { - return nil, err - } - return repo.Stat(rev) + return r.repo().Zip(dst, version) } -// InfoFile is like Stat but returns the name of the file containing -// the cached information. +// InfoFile is like Lookup(path).Stat(version) but returns the name of the file +// containing the cached information. func InfoFile(path, version string) (string, error) { if !semver.IsValid(version) { return "", fmt.Errorf("invalid version %q", version) @@ -307,10 +308,7 @@ func InfoFile(path, version string) (string, error) { } err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - _, err = repo.Stat(version) - } + _, err := Lookup(proxy, path).Stat(version) return err }) if err != nil { @@ -336,11 +334,7 @@ func GoMod(path, rev string) ([]byte, error) { rev = info.Version } else { err := TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err != nil { - return err - } - info, err := repo.Stat(rev) + info, err := Lookup(proxy, path).Stat(rev) if err == nil { rev = info.Version } @@ -357,11 +351,8 @@ func GoMod(path, rev string) ([]byte, error) { return data, nil } - err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, path) - if err == nil { - data, err = repo.GoMod(rev) - } + err = TryProxies(func(proxy string) (err error) { + data, err = Lookup(proxy, path).GoMod(rev) return err }) return data, err diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go index d85eddf767..c5fbb31b2b 100644 --- a/src/cmd/go/internal/modfetch/codehost/codehost.go +++ b/src/cmd/go/internal/modfetch/codehost/codehost.go @@ -11,6 +11,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/fs" "io/ioutil" "os" "os/exec" @@ -79,9 +80,8 @@ type Repo interface { ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) // RecentTag returns the most recent tag on rev or one of its predecessors - // with the given prefix and major version. - // An empty major string matches any major version. - RecentTag(rev, prefix, major string) (tag string, err error) + // with the given prefix. allowed may be used to filter out unwanted versions. + RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) // DescendsFrom reports whether rev or any of its ancestors has the given tag. // @@ -106,7 +106,7 @@ type FileRev struct { Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev } -// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a +// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a // revision rather than a file. type UnknownRevisionError struct { Rev string @@ -116,10 +116,10 @@ func (e *UnknownRevisionError) Error() string { return "unknown revision " + e.Rev } func (UnknownRevisionError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } -// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given +// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given // repository or module contains no commits. var ErrNoCommits error = noCommitsError{} @@ -129,7 +129,7 @@ func (noCommitsError) Error() string { return "no commits" } func (noCommitsError) Is(err error) bool { - return err == os.ErrNotExist + return err == fs.ErrNotExist } // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go index 31921324a7..8abc039e7f 100644 --- a/src/cmd/go/internal/modfetch/codehost/git.go +++ b/src/cmd/go/internal/modfetch/codehost/git.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" "os" "os/exec" @@ -34,13 +34,13 @@ func LocalGitRepo(remote string) (Repo, error) { } // A notExistError wraps another error to retain its original text -// but makes it opaquely equivalent to os.ErrNotExist. +// but makes it opaquely equivalent to fs.ErrNotExist. type notExistError struct { err error } func (e notExistError) Error() string { return e.err.Error() } -func (notExistError) Is(err error) bool { return err == os.ErrNotExist } +func (notExistError) Is(err error) bool { return err == fs.ErrNotExist } const gitWorkDirType = "git3" @@ -188,7 +188,7 @@ func (r *gitRepo) loadRefs() { // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL // ourselves and see what code it serves. if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { - if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) { + if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { gitErr = notExistError{gitErr} } } @@ -505,7 +505,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { } out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -629,9 +629,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F case "tag", "commit": switch fileType { default: - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} case "missing": - f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist} + f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist} case "blob": f.Data = fileData } @@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F return missing, nil } -func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { info, err := r.Stat(rev) if err != nil { return "", err @@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) { // NOTE: Do not replace the call to semver.Compare with semver.Max. // We want to return the actual tag, not a canonicalized version of it, // and semver.Max currently canonicalizes (see golang.org/issue/32700). - if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 { + if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) { + continue + } + if semver.Compare(semtag, highest) > 0 { highest = semtag } } @@ -823,12 +826,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args) if err != nil { if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return nil, err } - return ioutil.NopCloser(bytes.NewReader(archive)), nil + return io.NopCloser(bytes.NewReader(archive)), nil } // ensureGitAttributes makes sure export-subst and export-ignore features are @@ -859,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) { } }() - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return err } diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go index ba27c70f5a..981e3fe91f 100644 --- a/src/cmd/go/internal/modfetch/codehost/git_test.go +++ b/src/cmd/go/internal/modfetch/codehost/git_test.go @@ -10,6 +10,8 @@ import ( "flag" "fmt" "internal/testenv" + "io" + "io/fs" "io/ioutil" "log" "os" @@ -210,7 +212,7 @@ var readFileTests = []struct { repo: gitrepo1, rev: "v2.3.4", file: "another.txt", - err: os.ErrNotExist.Error(), + err: fs.ErrNotExist.Error(), }, } @@ -432,7 +434,7 @@ func TestReadZip(t *testing.T) { if tt.err != "" { t.Fatalf("ReadZip: no error, wanted %v", tt.err) } - zipdata, err := ioutil.ReadAll(rc) + zipdata, err := io.ReadAll(rc) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/codehost/shell.go b/src/cmd/go/internal/modfetch/codehost/shell.go index 2762c55720..b9525adf5e 100644 --- a/src/cmd/go/internal/modfetch/codehost/shell.go +++ b/src/cmd/go/internal/modfetch/codehost/shell.go @@ -14,6 +14,7 @@ import ( "bytes" "flag" "fmt" + "io" "io/ioutil" "log" "os" @@ -115,7 +116,7 @@ func main() { fmt.Fprintf(os.Stderr, "?%s\n", err) continue } - data, err := ioutil.ReadAll(rc) + data, err := io.ReadAll(rc) rc.Close() if err != nil { fmt.Fprintf(os.Stderr, "?%s\n", err) diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go index 7284557f4b..ec97fc7e1b 100644 --- a/src/cmd/go/internal/modfetch/codehost/vcs.go +++ b/src/cmd/go/internal/modfetch/codehost/vcs.go @@ -9,6 +9,7 @@ import ( "fmt" "internal/lazyregexp" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -377,7 +378,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) if err != nil { - return nil, os.ErrNotExist + return nil, fs.ErrNotExist } return out, nil } @@ -395,7 +396,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s return nil, vcsErrorf("ReadFileRevs not implemented") } -func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) { +func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in // lock-inversion bugs. diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go index d043903336..b6bcf83f1a 100644 --- a/src/cmd/go/internal/modfetch/coderepo.go +++ b/src/cmd/go/internal/modfetch/coderepo.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path" @@ -419,9 +420,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e tagPrefix = r.codeDir + "/" } + isRetracted, err := r.retractedVersions() + if err != nil { + isRetracted = func(string) bool { return false } + } + // tagToVersion returns the version obtained by trimming tagPrefix from tag. - // If the tag is invalid or a pseudo-version, tagToVersion returns an empty - // version. + // If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns + // an empty version. tagToVersion := func(tag string) (v string, tagIsCanonical bool) { if !strings.HasPrefix(tag, tagPrefix) { return "", false @@ -436,6 +442,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e if v == "" || !strings.HasPrefix(trimmed, v) { return "", false // Invalid or incomplete version (just vX or vX.Y). } + if isRetracted(v) { + return "", false + } if v == trimmed { tagIsCanonical = true } @@ -500,15 +509,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return checkGoMod() } + // Find the highest tagged version in the revision's history, subject to + // major version and +incompatible constraints. Use that version as the + // pseudo-version base so that the pseudo-version sorts higher. Ignore + // retracted versions. + allowedMajor := func(major string) func(v string) bool { + return func(v string) bool { + return (major == "" || semver.Major(v) == major) && !isRetracted(v) + } + } if pseudoBase == "" { var tag string if r.pseudoMajor != "" || canUseIncompatible() { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor) + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor)) } else { // Allow either v1 or v0, but not incompatible higher versions. - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1")) if tag == "" { - tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0") + tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0")) } } pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid @@ -869,6 +887,57 @@ func (r *codeRepo) modPrefix(rev string) string { return r.modPath + "@" + rev } +func (r *codeRepo) retractedVersions() (func(string) bool, error) { + versions, err := r.Versions("") + if err != nil { + return nil, err + } + + for i, v := range versions { + if strings.HasSuffix(v, "+incompatible") { + versions = versions[:i] + break + } + } + if len(versions) == 0 { + return func(string) bool { return false }, nil + } + + var highest string + for i := len(versions) - 1; i >= 0; i-- { + v := versions[i] + if semver.Prerelease(v) == "" { + highest = v + break + } + } + if highest == "" { + highest = versions[len(versions)-1] + } + + data, err := r.GoMod(highest) + if err != nil { + return nil, err + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, err + } + retractions := make([]modfile.VersionInterval, len(f.Retract)) + for _, r := range f.Retract { + retractions = append(retractions, r.VersionInterval) + } + + return func(v string) bool { + for _, r := range retractions { + if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 { + return true + } + } + return false + }, nil +} + func (r *codeRepo) Zip(dst io.Writer, version string) error { if version != module.CanonicalVersion(version) { return fmt.Errorf("version %s is not canonical", version) @@ -972,7 +1041,7 @@ type zipFile struct { } func (f zipFile) Path() string { return f.name } -func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil } +func (f zipFile) Lstat() (fs.FileInfo, error) { return f.f.FileInfo(), nil } func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() } type dataFile struct { @@ -981,9 +1050,9 @@ type dataFile struct { } func (f dataFile) Path() string { return f.name } -func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil } +func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil } func (f dataFile) Open() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(f.data)), nil + return io.NopCloser(bytes.NewReader(f.data)), nil } type dataFileInfo struct { @@ -992,7 +1061,7 @@ type dataFileInfo struct { func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) } func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } -func (fi dataFileInfo) Mode() os.FileMode { return 0644 } +func (fi dataFileInfo) Mode() fs.FileMode { return 0644 } func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } func (fi dataFileInfo) IsDir() bool { return false } func (fi dataFileInfo) Sys() interface{} { return nil } diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go index f69c193b86..4364fef6d1 100644 --- a/src/cmd/go/internal/modfetch/coderepo_test.go +++ b/src/cmd/go/internal/modfetch/coderepo_test.go @@ -60,7 +60,6 @@ var altVgotests = map[string]string{ type codeRepoTest struct { vcs string path string - lookErr string mpath string rev string err string @@ -332,9 +331,9 @@ var codeRepoTests = []codeRepoTest{ // package in subdirectory - custom domain // In general we can't reject these definitively in Lookup, // but gopkg.in is special. - vcs: "git", - path: "gopkg.in/yaml.v2/abc", - lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"", + vcs: "git", + path: "gopkg.in/yaml.v2/abc", + err: "invalid module path \"gopkg.in/yaml.v2/abc\"", }, { // package in subdirectory - github @@ -440,16 +439,7 @@ func TestCodeRepo(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if tt.lookErr != "" { - if err != nil && err.Error() == tt.lookErr { - return - } - t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr) - } - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) if tt.mpath == "" { tt.mpath = tt.path @@ -658,7 +648,7 @@ var codeRepoVersionsTests = []struct { { vcs: "git", path: "gopkg.in/russross/blackfriday.v2", - versions: []string{"v2.0.0", "v2.0.1"}, + versions: []string{"v2.0.0", "v2.0.1", "v2.1.0-pre.1"}, }, { vcs: "git", @@ -685,10 +675,7 @@ func TestCodeRepoVersions(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) list, err := repo.Versions(tt.prefix) if err != nil { t.Fatalf("Versions(%q): %v", tt.prefix, err) @@ -763,10 +750,7 @@ func TestLatest(t *testing.T) { testenv.MustHaveExecPath(t, tt.vcs) } - repo, err := Lookup("direct", tt.path) - if err != nil { - t.Fatalf("Lookup(%q): %v", tt.path, err) - } + repo := Lookup("direct", tt.path) info, err := repo.Latest() if err != nil { if tt.err != "" { diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 01d8f007ac..25e9fb62c1 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -63,14 +64,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) defer span.Done() - // If the directory exists, and no .partial file exists, the module has - // already been completely extracted. .partial files may be created when a - // module zip directory is extracted in place instead of being extracted to a - // temporary directory and renamed. dir, err = DownloadDir(mod) if err == nil { + // The directory has already been completely extracted (no .partial file exists). return dir, nil - } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + } else if dir == "" || !errors.Is(err, fs.ErrNotExist) { return "", err } @@ -88,6 +86,9 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { } defer unlock() + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) + defer span.Done() + // Check whether the directory was populated while we were waiting on the lock. _, dirErr := DownloadDir(mod) if dirErr == nil { @@ -95,10 +96,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { } _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs, as well - // as partially extracted diectories created by future versions of cmd/go. - // This is only safe to do because the lock file ensures that their writers - // are no longer active. + // Clean up any remaining temporary directories created by old versions + // (before 1.16), as well as partially extracted directories (indicated by + // DownloadDirPartialError, usually because of a .partial file). This is only + // safe to do because the lock file ensures that their writers are no longer + // active. parentDir := filepath.Dir(dir) tmpPrefix := filepath.Base(dir) + ".tmp-" if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil { @@ -116,88 +118,44 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { if err != nil { return "", err } - if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { - return "", err - } - // Extract the module zip directory. + // Extract the module zip directory at its final location. // - // By default, we extract to a temporary directory, then atomically rename to - // its final location. We use the existence of the source directory to signal - // that it has been extracted successfully (see DownloadDir). If someone - // deletes the entire directory (e.g., as an attempt to prune out file - // corruption), the module cache will still be left in a recoverable - // state. + // To prevent other processes from reading the directory if we crash, + // create a .partial file before extracting the directory, and delete + // the .partial file afterward (all while holding the lock). // - // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if - // another process opens files in the temporary directory. This is partially - // mitigated by using robustio.Rename, which retries os.Rename for a short - // time. + // Before Go 1.16, we extracted to a temporary directory with a random name + // then renamed it into place with os.Rename. On Windows, this failed with + // ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner) + // opened files in the temporary directory. // - // To avoid this error completely, if unzipInPlace is set, we instead create a - // .partial file (indicating the directory isn't fully extracted), then we - // extract the directory at its final location, then we delete the .partial - // file. This is not the default behavior because older versions of Go may - // simply stat the directory to check whether it exists without looking for a - // .partial file. If multiple versions run concurrently, the older version may - // assume a partially extracted directory is complete. - // TODO(golang.org/issue/36568): when these older versions are no longer - // supported, remove the old default behavior and the unzipInPlace flag. + // Go 1.14.2 and higher respect .partial files. Older versions may use + // partially extracted directories. 'go mod verify' can detect this, + // and 'go clean -modcache' can fix it. if err := os.MkdirAll(parentDir, 0777); err != nil { return "", err } - - ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) - if unzipInPlace { - if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { - return "", err - } - if err := modzip.Unzip(dir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - if rmErr := RemoveAll(dir); rmErr == nil { - os.Remove(partialPath) - } - return "", err - } - if err := os.Remove(partialPath); err != nil { - return "", err - } - } else { - tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) - if err != nil { - return "", err - } - if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { - fmt.Fprintf(os.Stderr, "-> %s\n", err) - RemoveAll(tmpDir) - return "", err - } - if err := robustio.Rename(tmpDir, dir); err != nil { - RemoveAll(tmpDir) - return "", err + if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { + return "", err + } + if err := modzip.Unzip(dir, mod, zipfile); err != nil { + fmt.Fprintf(os.Stderr, "-> %s\n", err) + if rmErr := RemoveAll(dir); rmErr == nil { + os.Remove(partialPath) } + return "", err + } + if err := os.Remove(partialPath); err != nil { + return "", err } - defer span.Done() if !cfg.ModCacheRW { - // Make dir read-only only *after* renaming it. - // os.Rename was observed to fail for read-only directories on macOS. makeDirsReadOnly(dir) } return dir, nil } -var unzipInPlace bool - -func init() { - for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { - if f == "modcacheunzipinplace=1" { - unzipInPlace = true - break - } - } -} - var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the @@ -276,12 +234,28 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e } }() + var unrecoverableErr error err = TryProxies(func(proxy string) error { - repo, err := Lookup(proxy, mod.Path) + if unrecoverableErr != nil { + return unrecoverableErr + } + repo := Lookup(proxy, mod.Path) + err := repo.Zip(f, mod.Version) if err != nil { - return err + // Zip may have partially written to f before failing. + // (Perhaps the server crashed while sending the file?) + // Since we allow fallback on error in some cases, we need to fix up the + // file to be empty again for the next attempt. + if _, err := f.Seek(0, io.SeekStart); err != nil { + unrecoverableErr = err + return err + } + if err := f.Truncate(0); err != nil { + unrecoverableErr = err + return err + } } - return repo.Zip(f, mod.Version) + return err }) if err != nil { return err @@ -341,10 +315,10 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e func makeDirsReadOnly(dir string) { type pathMode struct { path string - mode os.FileMode + mode fs.FileMode } var dirs []pathMode // in lexical order - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err == nil && info.Mode()&0222 != 0 { if info.IsDir() { dirs = append(dirs, pathMode{path, info.Mode()}) @@ -363,7 +337,7 @@ func makeDirsReadOnly(dir string) { // any permission changes needed to do so. func RemoveAll(dir string) error { // Module cache has 0555 directories; make them writable in order to remove content. - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { return nil // ignore errors walking in file system } @@ -454,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error return nil } +// HaveSum returns true if the go.sum file contains an entry for mod. +// The entry's hash must be generated with a known hash algorithm. +// mod.Version may have a "/go.mod" suffix to distinguish sums for +// .mod and .zip files. +func HaveSum(mod module.Version) bool { + goSum.mu.Lock() + defer goSum.mu.Unlock() + inited, err := initGoSum() + if err != nil || !inited { + return false + } + for _, h := range goSum.m[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + return false +} + // checkMod checks the given module's checksum. func checkMod(mod module.Version) { if cfg.GOMODCACHE == "" { @@ -468,7 +464,7 @@ func checkMod(mod module.Version) { } data, err := renameio.ReadFile(ziphash) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. return } @@ -487,7 +483,7 @@ func checkMod(mod module.Version) { // goModSum returns the checksum for the go.mod contents. func goModSum(data []byte) (string, error) { return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil }) } diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 4ac26650a9..d75b4da521 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -9,9 +9,8 @@ import ( "errors" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "path" pathpkg "path" "path/filepath" @@ -186,7 +185,7 @@ func proxyList() ([]proxySpec, error) { // TryProxies iterates f over each configured proxy (including "noproxy" and // "direct" if applicable) until f returns no error or until f returns an -// error that is not equivalent to os.ErrNotExist on a proxy configured +// error that is not equivalent to fs.ErrNotExist on a proxy configured // not to fall back on errors. // // TryProxies then returns that final error. @@ -222,7 +221,7 @@ func TryProxies(f func(proxy string) error) error { if err == nil { return nil } - isNotExistErr := errors.Is(err, os.ErrNotExist) + isNotExistErr := errors.Is(err, fs.ErrNotExist) if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) { bestErr = err @@ -305,7 +304,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) { return nil, err } defer body.Close() - return ioutil.ReadAll(body) + return io.ReadAll(body) } func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { @@ -428,7 +427,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { func (p *proxyRepo) Latest() (*RevInfo, error) { data, err := p.getBytes("@latest") if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { return nil, p.versionError("", err) } return p.latest() diff --git a/src/cmd/go/internal/modfetch/pseudo.go b/src/cmd/go/internal/modfetch/pseudo.go index 20c0b060ab..93eb0fad96 100644 --- a/src/cmd/go/internal/modfetch/pseudo.go +++ b/src/cmd/go/internal/modfetch/pseudo.go @@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string { return v + incDecimal(patch) + "-0." + segment + build } +// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and +// revision, which may be used as a placeholder. +func ZeroPseudoVersion(major string) string { + return PseudoVersion(major, "", time.Time{}, "000000000000") +} + // incDecimal returns the decimal string incremented by 1. func incDecimal(decimal string) string { // Scan right to left turning 9s to 0s until you find a digit to increment. @@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool { return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v) } +// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, +// timestamp, and revision, as returned by ZeroPseudoVersion. +func IsZeroPseudoVersion(v string) bool { + return v == ZeroPseudoVersion(semver.Major(v)) +} + // PseudoVersionTime returns the time stamp of the pseudo-version v. // It returns an error if v is not a pseudo-version or if the time stamp // embedded in the pseudo-version is not a valid time. diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index eed4dd4258..af9e24cefd 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -7,6 +7,7 @@ package modfetch import ( "fmt" "io" + "io/fs" "os" "sort" "strconv" @@ -32,8 +33,17 @@ type Repo interface { // Versions lists all known versions with the given prefix. // Pseudo-versions are not included. + // // Versions should be returned sorted in semver order // (implementations can use SortVersions). + // + // Versions returns a non-nil error only if there was a problem + // fetching the list of versions: it may return an empty list + // along with a nil error if the list of matching versions + // is known to be empty. + // + // If the underlying repository does not exist, + // Versions returns an error matching errors.Is(_, os.NotExist). Versions(prefix string) ([]string, error) // Stat returns information about the revision rev. @@ -188,27 +198,26 @@ type lookupCacheKey struct { // // A successful return does not guarantee that the module // has any defined versions. -func Lookup(proxy, path string) (Repo, error) { +func Lookup(proxy, path string) Repo { if traceRepo { defer logCall("Lookup(%q, %q)", proxy, path)() } type cached struct { - r Repo - err error + r Repo } c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { - r, err := lookup(proxy, path) - if err == nil { - if traceRepo { + r := newCachingRepo(path, func() (Repo, error) { + r, err := lookup(proxy, path) + if err == nil && traceRepo { r = newLoggingRepo(r) } - r = newCachingRepo(r) - } - return cached{r, err} + return r, err + }) + return cached{r} }).(cached) - return c.r, c.err + return c.r } // lookup returns the module with the given module path. @@ -228,7 +237,7 @@ func lookup(proxy, path string) (r Repo, err error) { switch proxy { case "off": - return nil, errProxyOff + return errRepo{path, errProxyOff}, nil case "direct": return lookupDirect(path) case "noproxy": @@ -407,7 +416,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error { return l.r.Zip(dst, version) } -// A notExistError is like os.ErrNotExist, but with a custom message +// errRepo is a Repo that returns the same error for all operations. +// +// It is useful in conjunction with caching, since cache hits will not attempt +// the prohibited operations. +type errRepo struct { + modulePath string + err error +} + +func (r errRepo) ModulePath() string { return r.modulePath } + +func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err } +func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err } +func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err } +func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err } +func (r errRepo) Zip(dst io.Writer, version string) error { return r.err } + +// A notExistError is like fs.ErrNotExist, but with a custom message type notExistError struct { err error } @@ -421,7 +447,7 @@ func (e notExistError) Error() string { } func (notExistError) Is(target error) bool { - return target == os.ErrNotExist + return target == fs.ErrNotExist } func (e notExistError) Unwrap() error { diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 47a2571531..4fbc54d15c 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -12,7 +12,8 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" + "io" + "io/fs" "net/url" "os" "path/filepath" @@ -182,7 +183,7 @@ func (c *dbClient) initBase() { return nil } }) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // No proxies, or all proxies failed (with 404, 410, or were were allowed // to fall back), or we reached an explicit "direct" or "off". c.base = c.direct @@ -203,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) { } targ := filepath.Join(cfg.SumdbDir, file) data, err = lockedfile.Read(targ) - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Treat non-existent as empty, to bootstrap the "latest" file // the first time we connect to a given database. return []byte{}, nil @@ -227,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error { return err } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } @@ -257,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) { // during which the empty file can be locked for reading. // Treat observing an empty file as file not found. if err == nil && len(data) == 0 { - err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist} + err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist} } return data, err } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index f1cf8b17a8..171c070ab3 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -115,13 +115,12 @@ require downgrading other dependencies, and 'go get' does this automatically as well. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. +custom domains using insecure schemes such as HTTP, and also bypassess +module sum validation using the checksum database. Use with caution. This flag is deprecated and will be removed in a future version of go. -The GOINSECURE environment variable is usually a better alternative, since -it provides control over which modules may be retrieved using an insecure -scheme. It should be noted that the -insecure flag also turns the module -checksum validation off. GOINSECURE does not do that, use GONOSUMDB. -See 'go help environment' for details. +To permit the use of insecure schemes, use the GOINSECURE environment +variable instead. To bypass module sum validation, use GOPRIVATE or +GONOSUMDB. See 'go help environment' for details. The second step is to download (if needed), build, and install the named packages. @@ -875,6 +874,8 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc allowed := modload.CheckAllowed if modload.IsRevisionQuery(vers) { allowed = modload.CheckExclusions + } else if vers == "upgrade" || vers == "patch" { + allowed = checkAllowedOrCurrent(prevM.Version) } // If the query must be a module path, try only that module path. @@ -911,7 +912,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Selected, allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -982,3 +983,18 @@ func logOncef(format string, args ...interface{}) { fmt.Fprintln(os.Stderr, msg) } } + +// checkAllowedOrCurrent is like modload.CheckAllowed, but always allows the +// current version (even if it is retracted or otherwise excluded). +func checkAllowedOrCurrent(current string) modload.AllowedFunc { + if current == "" { + return modload.CheckAllowed + } + + return func(ctx context.Context, m module.Version) error { + if m.Version == current { + return nil + } + return modload.CheckAllowed(ctx, m) + } +} diff --git a/src/cmd/go/internal/modget/mvs.go b/src/cmd/go/internal/modget/mvs.go index 19fffd2947..e7e0ec80d0 100644 --- a/src/cmd/go/internal/modget/mvs.go +++ b/src/cmd/go/internal/modget/mvs.go @@ -145,7 +145,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.CheckAllowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, checkAllowedOrCurrent(m.Version)) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index f49b52df56..b9abb0b93c 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -13,7 +13,6 @@ import ( "internal/goroot" "os" "path/filepath" - "runtime/debug" "strings" "cmd/go/internal/base" @@ -312,9 +311,6 @@ func mustFindModule(target, path string) module.Version { return Target } - if printStackInDie { - debug.PrintStack() - } base.Fatalf("build %v: cannot find module for path %v", target, path) panic("unreachable") } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 059b020420..4a183d6881 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -37,7 +37,7 @@ var buildList []module.Version // // The caller must not modify the returned list. func LoadAllModules(ctx context.Context) []module.Version { - InitMod(ctx) + LoadModFile(ctx) ReloadBuildList() WriteGoMod() return buildList @@ -52,6 +52,21 @@ func LoadedModules() []module.Version { return buildList } +// Selected returns the selected version of the module with the given path, or +// the empty string if the given module has no selected version +// (either because it is not required or because it is the Target module). +func Selected(path string) (version string) { + if path == Target.Path { + return "" + } + for _, m := range buildList { + if m.Path == path { + return m.Version + } + } + return "" +} + // SetBuildList sets the module build list. // The caller is responsible for ensuring that the list is valid. // SetBuildList does not retain a reference to the original list. @@ -63,7 +78,9 @@ func SetBuildList(list []module.Version) { // the build list set in SetBuildList. func ReloadBuildList() []module.Version { loaded = loadFromRoots(loaderParams{ - tags: imports.Tags(), + PackageOpts: PackageOpts{ + Tags: imports.Tags(), + }, listRoots: func() []string { return nil }, allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. }) diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index c36c8bd29b..e959347020 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -10,13 +10,14 @@ import ( "fmt" "go/build" "internal/goroot" + "io/fs" "os" "path/filepath" "sort" "strings" - "time" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modfetch" "cmd/go/internal/par" "cmd/go/internal/search" @@ -25,13 +26,25 @@ import ( "golang.org/x/mod/semver" ) -var errImportMissing = errors.New("import missing") - type ImportMissingError struct { Path string Module module.Version QueryErr error + // inAll indicates whether Path is in the "all" package pattern, + // and thus would be added by 'go mod tidy'. + inAll bool + + // isStd indicates whether we would expect to find the package in the standard + // library. This is normally true for all dotless import paths, but replace + // directives can cause us to treat the replaced paths as also being in + // modules. + isStd bool + + // replaced the highest replaced version of the module where the replacement + // contains the package. replaced is only set if the replacement is unused. + replaced module.Version + // newMissingVersion is set to a newer version of Module if one is present // in the build list. When set, we can't automatically upgrade. newMissingVersion string @@ -39,13 +52,33 @@ type ImportMissingError struct { func (e *ImportMissingError) Error() string { if e.Module.Path == "" { - if search.IsStandardImportPath(e.Path) { + if e.isStd { return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path)) } if e.QueryErr != nil { return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr) } - return "cannot find module providing package " + e.Path + if cfg.BuildMod == "mod" { + return "cannot find module providing package " + e.Path + } + + if e.replaced.Path != "" { + suggestArg := e.replaced.Path + if !modfetch.IsZeroPseudoVersion(e.replaced.Version) { + suggestArg = e.replaced.String() + } + return fmt.Sprintf("module %s provides package %s and is replaced but not required; try 'go get -d %s' to add it", e.replaced.Path, e.Path, suggestArg) + } + + suggestion := "" + if !HasModRoot() { + suggestion = ": working directory is not part of a module" + } else if e.inAll { + suggestion = "; try 'go mod tidy' to add it" + } else { + suggestion = fmt.Sprintf("; try 'go get -d %s' to add it", e.Path) + } + return fmt.Sprintf("no required module provides package %s%s", e.Path, suggestion) } if e.newMissingVersion != "" { @@ -102,6 +135,31 @@ func (e *AmbiguousImportError) Error() string { return buf.String() } +// ImportMissingSumError is reported in readonly mode when we need to check +// if a module in the build list contains a package, but we don't have a sum +// for its .zip file. +type ImportMissingSumError struct { + importPath string + found, inAll bool +} + +func (e *ImportMissingSumError) Error() string { + var message string + if e.found { + message = fmt.Sprintf("missing go.sum entry needed to verify package %s is provided by exactly one module", e.importPath) + } else { + message = fmt.Sprintf("missing go.sum entry for module providing package %s", e.importPath) + } + if e.inAll { + return message + "; try 'go mod tidy' to add it" + } + return message +} + +func (e *ImportMissingSumError) ImportPath() string { + return e.importPath +} + type invalidImportError struct { importPath string err error @@ -131,7 +189,7 @@ func (e *invalidImportError) Unwrap() error { // like "C" and "unsafe". // // If the package cannot be found in the current build list, -// importFromBuildList returns errImportMissing as the error. +// importFromBuildList returns an *ImportMissingError. func importFromBuildList(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") @@ -143,6 +201,10 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di // There's no directory for import "C" or import "unsafe". return module.Version{}, "", nil } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, "", &invalidImportError{importPath: path, err: err} + } // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) @@ -182,13 +244,23 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di // Check each module on the build list. var dirs []string var mods []module.Version + haveSumErr := false for _, m := range buildList { if !maybeInModule(path, m.Path) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(ctx, m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + // We are missing a sum needed to fetch a module in the build list. + // We can't verify that the package is unique, and we may not find + // the package at all. Keep checking other modules to decide which + // error to report. + haveSumErr = true + continue + } // Report fetch error. // Note that we don't know for sure this module is necessary, // but it certainly _could_ provide the package, and even if we @@ -204,60 +276,46 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di dirs = append(dirs, dir) } } + if len(mods) > 1 { + return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + } + if haveSumErr { + return module.Version{}, "", &ImportMissingSumError{importPath: path, found: len(mods) > 0} + } if len(mods) == 1 { return mods[0], dirs[0], nil } - if len(mods) > 0 { - return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} - } - return module.Version{}, "", errImportMissing + return module.Version{}, "", &ImportMissingError{Path: path, isStd: pathIsStd} } // queryImport attempts to locate a module that can be added to the current // build list to provide the package with the given import path. +// +// Unlike QueryPattern, queryImport prefers to add a replaced version of a +// module *before* checking the proxies for a version to add. func queryImport(ctx context.Context, path string) (module.Version, error) { - pathIsStd := search.IsStandardImportPath(path) - - if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, &ImportMissingError{ - Path: path, - QueryErr: errors.New("working directory is not part of a module"), - } - } - - // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each - // module (golang.org/issue/26241). This should give a useful message - // in -mod=readonly, and it will allow us to add a requirement with -mod=mod. - if modFile != nil { - latest := map[string]string{} // path -> version - for _, r := range modFile.Replace { - if maybeInModule(path, r.Old.Path) { - // Don't use semver.Max here; need to preserve +incompatible suffix. - v := latest[r.Old.Path] - if semver.Compare(r.Old.Version, v) > 0 { - v = r.Old.Version - } - latest[r.Old.Path] = v + // To avoid spurious remote fetches, try the latest replacement for each + // module (golang.org/issue/26241). + if index != nil { + var mods []module.Version + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue } - } - - mods := make([]module.Version, 0, len(latest)) - for p, v := range latest { - // If the replacement didn't specify a version, synthesize a - // pseudo-version with an appropriate major version and a timestamp below - // any real timestamp. That way, if the main module is used from within - // some other module, the user will be able to upgrade the requirement to - // any real version they choose. - if v == "" { - if _, pathMajor, ok := module.SplitPathVersion(p); ok && len(pathMajor) > 0 { - v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = modfetch.ZeroPseudoVersion(pathMajor[1:]) } else { - v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + mv = modfetch.ZeroPseudoVersion("v0") } } - mods = append(mods, module.Version{Path: p, Version: v}) + mods = append(mods, module.Version{Path: mp, Version: mv}) } // Every module path in mods is a prefix of the import path. @@ -266,20 +324,26 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(ctx, m) + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { - // Report fetch error as above. + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} + } return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { return m, err } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} + } return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, - // so it can only exist if in a replaced module, + // so it can only exist in a replaced module, // and we know from the above loop that it is not. return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], @@ -290,12 +354,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { } } - // Before any further lookup, check that the path is valid. - if err := module.CheckImportPath(path); err != nil { - return module.Version{}, &invalidImportError{importPath: path, err: err} - } - - if pathIsStd { + if search.IsStandardImportPath(path) { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has // shimmed in via a "replace" directive. @@ -303,10 +362,13 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // QueryPattern cannot possibly find a module containing this package. // // Instead of trying QueryPattern, report an ImportMissingError immediately. - return module.Version{}, &ImportMissingError{Path: path} + return module.Version{}, &ImportMissingError{Path: path, isStd: true} } if cfg.BuildMod == "readonly" { + // In readonly mode, we can't write go.mod, so we shouldn't try to look up + // the module. If readonly mode was enabled explicitly, include that in + // the error message. var queryErr error if cfg.BuildModExplicit { queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) @@ -321,9 +383,9 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPattern(ctx, path, "latest", CheckAllowed) + candidates, err := QueryPattern(ctx, path, "latest", Selected, CheckAllowed) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever // low-level error QueryPattern produced. return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} @@ -438,57 +500,65 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. res := haveGoFilesCache.Do(dir, func() interface{} { - ok, err := isDirWithGoFiles(dir) + ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) return dir, res.haveGoFiles, res.err } -func isDirWithGoFiles(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - defer f.Close() - - names, firstErr := f.Readdirnames(-1) - if firstErr != nil { - if fi, err := f.Stat(); err == nil && !fi.IsDir() { - return false, nil - } - - // Rewrite the error from ReadDirNames to include the path if not present. - // See https://golang.org/issue/38923. - var pe *os.PathError - if !errors.As(firstErr, &pe) { - firstErr = &os.PathError{Op: "readdir", Path: dir, Err: firstErr} - } +// fetch downloads the given module (or its replacement) +// and returns its location. +// +// needSum indicates whether the module may be downloaded in readonly mode +// without a go.sum entry. It should only be false for modules fetched +// speculatively (for example, for incompatible version filtering). The sum +// will still be verified normally. +// +// The isLocal return value reports whether the replacement, +// if any, is local to the filesystem. +func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { + if mod == Target { + return ModRoot(), true, nil } - - for _, name := range names { - if strings.HasSuffix(name, ".go") { - info, err := os.Stat(filepath.Join(dir, name)) - if err == nil && info.Mode().IsRegular() { - // If any .go source file exists, the package exists regardless of - // errors for other source files. Leave further error reporting for - // later. - return true, nil + if r := Replacement(mod); r.Path != "" { + if r.Version == "" { + dir = r.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) } - if firstErr == nil { + // Ensure that the replacement directory actually exists: + // dirInModule does not report errors for missing modules, + // so if we don't report the error now, later failures will be + // very mysterious. + if _, err := os.Stat(dir); err != nil { if os.IsNotExist(err) { - // If the file was concurrently deleted, or was a broken symlink, - // convert the error to an opaque error instead of one matching - // os.IsNotExist. - err = errors.New(err.Error()) + // Semantically the module version itself “exists” — we just don't + // have its source code. Remove the equivalence to os.ErrNotExist, + // and make the message more concise while we're at it. + err = fmt.Errorf("replacement directory %s does not exist", r.Path) + } else { + err = fmt.Errorf("replacement directory %s: %w", r.Path, err) } - firstErr = err + return dir, true, module.VersionError(mod, err) } + return dir, true, nil } + mod = r + } + + if cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + return "", false, module.VersionError(mod, &sumMissingError{}) } - return false, firstErr + dir, err = modfetch.Download(ctx, mod) + return dir, false, err +} + +type sumMissingError struct { + suggestion string +} + +func (e *sumMissingError) Error() string { + return "missing go.sum entry" + e.suggestion } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 3344242489..8fe71a2448 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -16,12 +16,13 @@ import ( "os" "path" "path/filepath" - "runtime/debug" "strconv" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/lockedfile" "cmd/go/internal/modconv" "cmd/go/internal/modfetch" @@ -50,9 +51,6 @@ var ( gopath string - CmdModInit bool // running 'go mod init' - CmdModModule string // module argument for 'go mod init' - // RootMode determines whether a module root is needed. RootMode Root @@ -132,6 +130,10 @@ func Init() { return } + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } + // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding // the prompt in earlier versions is just too hard. @@ -159,9 +161,9 @@ func Init() { os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - modRoot = base.Cwd + if modRoot != "" { + // modRoot set before Init was called ("go mod init" does this). + // No need to search for go.mod. } else if RootMode == NoRoot { if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") @@ -198,8 +200,7 @@ func Init() { base.Fatalf("go: -modfile=%s: file does not have .mod extension", cfg.ModFile) } - // We're in module mode. Install the hooks to make it work. - + // We're in module mode. Set any global variables that need to be set. list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") @@ -266,10 +267,6 @@ func WillBeEnabled() bool { return false } - if CmdModInit { - // Running 'go mod init': go.mod will be created in current directory. - return true - } if modRoot := findModuleRoot(base.Cwd); modRoot == "" { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -325,16 +322,7 @@ func ModFilePath() string { return filepath.Join(modRoot, "go.mod") } -// printStackInDie causes die to print a stack trace. -// -// It is enabled by the testgo tag, and helps to diagnose paths that -// unexpectedly require a main module. -var printStackInDie = false - func die() { - if printStackInDie { - debug.PrintStack() - } if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } @@ -352,16 +340,16 @@ func die() { base.Fatalf("go: cannot find main module; see 'go help modules'") } -// InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file. If InitMod is called by 'go mod init', InitMod -// will populate go.mod in memory, possibly importing dependencies from a -// legacy configuration file. For other commands, InitMod may make other -// adjustments in memory, like adding a go directive. WriteGoMod should be -// called later to write changes out to disk. +// LoadModFile sets Target and, if there is a main module, parses the initial +// build list from its go.mod file. // -// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not +// LoadModFile may make changes in memory, like adding a go directive and +// ensuring requirements are consistent. WriteGoMod should be called later to +// write changes out to disk or report errors in readonly mode. +// +// As a side-effect, LoadModFile sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod(ctx context.Context) { +func LoadModFile(ctx context.Context) { if len(buildList) > 0 { return } @@ -374,13 +362,6 @@ func InitMod(ctx context.Context) { return } - if CmdModInit { - // Running go mod init: do legacy module conversion - legacyModInit() - modFileToBuildList() - return - } - gomod := ModFilePath() data, err := lockedfile.Read(gomod) if err != nil { @@ -401,12 +382,6 @@ func InitMod(ctx context.Context) { base.Fatalf("go: no module declaration in go.mod.\n\tRun 'go mod edit -module=example.com/mod' to specify the module path.") } - if len(f.Syntax.Stmt) == 1 && f.Module != nil { - // Entire file is just a module statement. - // Populate require if possible. - legacyModInit() - } - if err := checkModulePathLax(f.Module.Mod.Path); err != nil { base.Fatalf("go: %v", err) } @@ -419,6 +394,73 @@ func InitMod(ctx context.Context) { } } +// CreateModFile initializes a new module by creating a go.mod file. +// +// If modPath is empty, CreateModFile will attempt to infer the path from the +// directory location within GOPATH. +// +// If a vendoring configuration file is present, CreateModFile will attempt to +// translate it to go.mod directives. The resulting build list may not be +// exactly the same as in the legacy configuration (for example, we can't get +// packages at multiple versions from the same module). +func CreateModFile(ctx context.Context, modPath string) { + modRoot = base.Cwd + Init() + modFilePath := ModFilePath() + if _, err := os.Stat(modFilePath); err == nil { + base.Fatalf("go: %s already exists", modFilePath) + } + + if modPath == "" { + var err error + modPath, err = findModulePath(modRoot) + if err != nil { + base.Fatalf("go: %v", err) + } + } else if err := checkModulePathLax(modPath); err != nil { + base.Fatalf("go: %v", err) + } + + fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) + modFile = new(modfile.File) + modFile.AddModuleStmt(modPath) + addGoStmt() // Add the go directive before converted module requirements. + + convertedFrom, err := convertLegacyConfig(modPath) + if convertedFrom != "" { + fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + modFileToBuildList() + WriteGoMod() + + // Suggest running 'go mod tidy' unless the project is empty. Even if we + // imported all the correct requirements above, we're probably missing + // some sums, so the next build command in -mod=readonly will likely fail. + // + // We look for non-hidden .go files or subdirectories to determine whether + // this is an existing project. Walking the tree for packages would be more + // accurate, but could take much longer. + empty := true + fis, _ := ioutil.ReadDir(modRoot) + for _, fi := range fis { + name := fi.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if strings.HasSuffix(name, ".go") || fi.IsDir() { + empty = false + break + } + } + if !empty { + fmt.Fprintf(os.Stderr, "go: run 'go mod tidy' to add module requirements and sums\n") + } +} + // checkModulePathLax checks that the path meets some minimum requirements // to avoid confusing users or the module cache. The requirements are weaker // than those of module.CheckPath to allow room for weakening module path @@ -585,38 +627,23 @@ func setDefaultBuildMod() { cfg.BuildMod = "readonly" } -func legacyModInit() { - if modFile == nil { - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", path) - modFile = new(modfile.File) - modFile.AddModuleStmt(path) - addGoStmt() // Add the go directive before converted module requirements. - } - +// convertLegacyConfig imports module requirements from a legacy vendoring +// configuration file, if one is present. +func convertLegacyConfig(modPath string) (from string, err error) { for _, name := range altConfigs { cfg := filepath.Join(modRoot, name) data, err := ioutil.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] if convert == nil { - return + return "", nil } - fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(cfg)) cfg = filepath.ToSlash(cfg) - if err := modconv.ConvertLegacyConfig(modFile, cfg, data); err != nil { - base.Fatalf("go: %v", err) - } - if len(modFile.Syntax.Stmt) == 1 { - // Add comment to avoid re-converting every time it runs. - modFile.AddComment("// go: no requirements found in " + name) - } - return + err := modconv.ConvertLegacyConfig(modFile, cfg, data) + return name, err } } + return "", nil } // addGoStmt adds a go directive to the go.mod file if it does not already include one. @@ -696,14 +723,6 @@ func findAltConfig(dir string) (root, name string) { } func findModulePath(dir string) (string, error) { - if CmdModModule != "" { - // Running go mod init x/y/z; return x/y/z. - if err := module.CheckImportPath(CmdModModule); err != nil { - return "", err - } - return CmdModModule, nil - } - // TODO(bcmills): once we have located a plausible module path, we should // query version control (if available) to verify that it matches the major // version of the most recent tag. @@ -893,7 +912,10 @@ func WriteGoMod() { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). // Don't write go.mod, but write go.sum in case we added or trimmed sums. - modfetch.WriteGoSum(keepSums(true)) + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } return } @@ -906,7 +928,10 @@ func WriteGoMod() { index = indexModFile(new, modFile, false) // Update go.sum after releasing the side lock and refreshing the index. - modfetch.WriteGoSum(keepSums(true)) + // 'go mod init' shouldn't write go.sum, since it will be incomplete. + if cfg.CmdName != "mod init" { + modfetch.WriteGoSum(keepSums(true)) + } }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -951,41 +976,55 @@ func WriteGoMod() { // If addDirect is true, the set also includes sums for modules directly // required by go.mod, as represented by the index, with replacements applied. func keepSums(addDirect bool) map[module.Version]bool { - // Walk the module graph and keep sums needed by MVS. + // Re-derive the build list using the current list of direct requirements. + // Keep the sum for the go.mod of each visited module version (or its + // replacement). modkey := func(m module.Version) module.Version { return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} } keep := make(map[module.Version]bool) - replaced := make(map[module.Version]bool) - reqs := Reqs() - var walk func(module.Version) - walk = func(m module.Version) { - // If we build using a replacement module, keep the sum for the replacement, - // since that's the code we'll actually use during a build. - r := Replacement(m) - if r.Path == "" { - keep[modkey(m)] = true - } else { - replaced[m] = true - keep[modkey(r)] = true - } - list, _ := reqs.Required(m) - for _, r := range list { - if !keep[modkey(r)] && !replaced[r] { - walk(r) + var mu sync.Mutex + reqs := &keepSumReqs{ + Reqs: Reqs(), + visit: func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + mu.Lock() + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + keep[modkey(r)] = true } - } + mu.Unlock() + }, + } + buildList, err := mvs.BuildList(Target, reqs) + if err != nil { + panic(fmt.Sprintf("unexpected error reloading build list: %v", err)) } - walk(Target) - // Add entries for modules from which packages were loaded. + // Add entries for modules in the build list with paths that are prefixes of + // paths of loaded packages. We need to retain sums for modules needed to + // report ambiguous import errors. We use our re-derived build list, + // since the global build list may have been tidied. if loaded != nil { - for _, pkg := range loaded.pkgs { - m := pkg.mod + actualMods := make(map[string]module.Version) + for _, m := range buildList[1:] { if r := Replacement(m); r.Path != "" { - keep[r] = true + actualMods[m.Path] = r } else { - keep[m] = true + actualMods[m.Path] = m + } + } + for _, pkg := range loaded.pkgs { + if pkg.testOf != nil || pkg.inStd { + continue + } + for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { + if m, ok := actualMods[prefix]; ok { + keep[m] = true + } } } } @@ -1007,6 +1046,18 @@ func keepSums(addDirect bool) map[module.Version]bool { return keep } +// keepSumReqs embeds another Reqs implementation. The Required method +// calls visit for each version in the module graph. +type keepSumReqs struct { + mvs.Reqs + visit func(module.Version) +} + +func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) { + r.visit(m) + return r.Reqs.Required(m) +} + func TrimGoSum() { // Don't retain sums for direct requirements in go.mod. When TrimGoSum is // called, go.mod has not been updated, and it may contain requirements on diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 9194f9cc7c..f9c468c8b2 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -98,6 +98,7 @@ import ( "errors" "fmt" "go/build" + "io/fs" "os" "path" pathpkg "path" @@ -111,6 +112,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/mvs" @@ -167,7 +169,7 @@ type PackageOpts struct { // LoadPackages identifies the set of packages matching the given patterns and // loads the packages in the import graph rooted at that set. func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { - InitMod(ctx) + LoadModFile(ctx) if opts.Tags == nil { opts.Tags = imports.Tags() } @@ -248,9 +250,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } loaded = loadFromRoots(loaderParams{ - tags: opts.Tags, - loadTests: opts.LoadTests, - resolveMissing: opts.ResolveMissingImports, + PackageOpts: opts, allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, allPatternIsRoot: allPatternIsRoot, @@ -270,11 +270,21 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // Report errors, if any. checkMultiplePaths() for _, pkg := range loaded.pkgs { - if pkg.err != nil && !opts.SilenceErrors { - if opts.AllowErrors { - fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) - } else { - base.Errorf("%s: %v", pkg.stackText(), pkg.err) + if pkg.err != nil { + if pkg.flags.has(pkgInAll) { + if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) { + imErr.inAll = true + } else if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) { + sumErr.inAll = true + } + } + + if !opts.SilenceErrors { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } } } if !pkg.isTest() { @@ -352,11 +362,11 @@ func resolveLocalPackage(dir string) (string, error) { // If the named directory does not exist or contains no Go files, // the package does not exist. // Other errors may affect package loading, but not resolution. - if _, err := os.Stat(absDir); err != nil { + if _, err := fsys.Stat(absDir); err != nil { if os.IsNotExist(err) { // Canonicalize OS-specific errors to errDirectoryNotFound so that error // messages will be easier for users to search for. - return "", &os.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} + return "", &fs.PathError{Op: "stat", Path: absDir, Err: errDirectoryNotFound} } return "", err } @@ -484,7 +494,7 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. func ImportFromFiles(ctx context.Context, gofiles []string) { - InitMod(ctx) + LoadModFile(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -493,8 +503,10 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { } loaded = loadFromRoots(loaderParams{ - tags: tags, - resolveMissing: true, + PackageOpts: PackageOpts{ + Tags: tags, + ResolveMissingImports: true, + }, allClosesOverTests: index.allPatternClosesOverTests(), listRoots: func() (roots []string) { roots = append(roots, imports...) @@ -647,10 +659,10 @@ type loader struct { direct map[string]bool // imported directly by main module } +// loaderParams configure the packages loaded by, and the properties reported +// by, a loader instance. type loaderParams struct { - tags map[string]bool // tags for scanDir - loadTests bool - resolveMissing bool + PackageOpts allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? allPatternIsRoot bool // Is the "all" pattern an additional root? @@ -809,7 +821,7 @@ func loadFromRoots(params loaderParams) *loader { ld.buildStacks() - if !ld.resolveMissing { + if !ld.ResolveMissingImports || (!HasModRoot() && !allowMissingModuleImports) { // We've loaded as much as we can without resolving missing imports. break } @@ -852,7 +864,7 @@ func loadFromRoots(params loaderParams) *loader { // contributes “direct” imports — so we can't safely mark existing // dependencies as indirect-only. // Conservatively mark those dependencies as direct. - if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.tags, imports.AnyTags())) { + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.Tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true @@ -872,12 +884,15 @@ func loadFromRoots(params loaderParams) *loader { func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { var needPkgs []*loadPkg for _, pkg := range ld.pkgs { + if pkg.err == nil { + continue + } if pkg.isTest() { // If we are missing a test, we are also missing its non-test version, and // we should only add the missing import once. continue } - if pkg.err != errImportMissing { + if !errors.As(pkg.err, new(*ImportMissingError)) { // Leave other errors for Import or load.Packages to report. continue } @@ -980,7 +995,7 @@ func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { // also in "all" (as above). wantTest = true - case ld.loadTests && new.has(pkgIsRoot): + case ld.LoadTests && new.has(pkgIsRoot): // LoadTest explicitly requests tests of “the root packages”. wantTest = true } @@ -1043,7 +1058,7 @@ func (ld *loader) load(pkg *loadPkg) { ld.applyPkgFlags(pkg, pkgInAll) } - imports, testImports, err := scanDir(pkg.dir, ld.tags) + imports, testImports, err := scanDir(pkg.dir, ld.Tags) if err != nil { pkg.err = err return diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 6457a7d968..7a8963246b 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -35,13 +35,14 @@ var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile // at a specific point in time. type modFileIndex struct { - data []byte - dataNeedsFix bool // true if fixVersion applied a change while parsing data - module module.Version - goVersionV string // GoVersion with "v" prefix - require map[module.Version]requireMeta - replace map[module.Version]module.Version - exclude map[module.Version]bool + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersionV string // GoVersion with "v" prefix + require map[module.Version]requireMeta + replace map[module.Version]module.Version + highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements + exclude map[module.Version]bool } // index is the index of the go.mod file as of when it was last read or written. @@ -115,9 +116,9 @@ func checkRetractions(ctx context.Context, m module.Version) error { // Ignore exclusions from the main module's go.mod. // We may need to account for the current version: for example, // v2.0.0+incompatible is not "latest" if v1.0.0 is current. - rev, err := Query(ctx, path, "latest", findCurrentVersion(path), nil) + rev, err := Query(ctx, path, "latest", Selected(path), nil) if err != nil { - return &entry{err: err} + return &entry{nil, err} } // Load go.mod for that version. @@ -138,13 +139,19 @@ func checkRetractions(ctx context.Context, m module.Version) error { } summary, err := rawGoModSummary(rm) if err != nil { - return &entry{err: err} + return &entry{nil, err} } - return &entry{retract: summary.retract} + return &entry{summary.retract, nil} }).(*entry) - if e.err != nil { - return fmt.Errorf("loading module retractions: %v", e.err) + if err := e.err; err != nil { + // Attribute the error to the version being checked, not the version from + // which the retractions were to be loaded. + var mErr *module.ModuleError + if errors.As(err, &mErr) { + err = mErr.Err + } + return &retractionLoadingError{m: m, err: err} } var rationale []string @@ -158,7 +165,7 @@ func checkRetractions(ctx context.Context, m module.Version) error { } } if isRetracted { - return &retractedError{rationale: rationale} + return module.VersionError(m, &retractedError{rationale: rationale}) } return nil } @@ -183,6 +190,19 @@ func (e *retractedError) Is(err error) bool { return err == ErrDisallowed } +type retractionLoadingError struct { + m module.Version + err error +} + +func (e *retractionLoadingError) Error() string { + return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err) +} + +func (e *retractionLoadingError) Unwrap() error { + return e.err +} + // ShortRetractionRationale returns a retraction rationale string that is safe // to print in a terminal. It returns hard-coded strings if the rationale // is empty, too long, or contains non-printable characters. @@ -255,6 +275,14 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.replace[r.Old] = r.New } + i.highestReplaced = make(map[string]string) + for _, r := range modFile.Replace { + v, ok := i.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + i.highestReplaced[r.Old.Path] = r.Old.Version + } + } + i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) for _, x := range modFile.Exclude { i.exclude[x.Mod] = true @@ -378,8 +406,11 @@ type retraction struct { // taking into account any replacements for m, exclusions of its dependencies, // and/or vendoring. // -// goModSummary cannot be used on the Target module, as its requirements -// may change. +// m must be a version in the module graph, reachable from the Target module. +// In readonly mode, the go.sum file must contain an entry for m's go.mod file +// (or its replacement). goModSummary must not be called for the Target module +// itself, as its requirements may change. Use rawGoModSummary for other +// module versions. // // The caller must not modify the returned summary. func goModSummary(m module.Version) (*modFileSummary, error) { @@ -414,6 +445,13 @@ func goModSummary(m module.Version) (*modFileSummary, error) { if actual.Path == "" { actual = m } + if cfg.BuildMod == "readonly" && actual.Version != "" { + key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} + if !modfetch.HaveSum(key) { + suggestion := fmt.Sprintf("; try 'go mod download %s' to add it", m.Path) + return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) + } + } summary, err := rawGoModSummary(actual) if err != nil { return nil, err diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 24856260d4..76a1d8a12a 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -7,9 +7,6 @@ package modload import ( "context" "errors" - "fmt" - "os" - "path/filepath" "sort" "cmd/go/internal/modfetch" @@ -77,11 +74,7 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return err - } - allVersions, err := repo.Versions("") + allVersions, err := modfetch.Lookup(proxy, path).Versions("") if err != nil { return err } @@ -129,42 +122,3 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { } return module.Version{Path: m.Path, Version: "none"}, nil } - -// fetch downloads the given module (or its replacement) -// and returns its location. -// -// The isLocal return value reports whether the replacement, -// if any, is local to the filesystem. -func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil - } - if r := Replacement(mod); r.Path != "" { - if r.Version == "" { - dir = r.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - // Ensure that the replacement directory actually exists: - // dirInModule does not report errors for missing modules, - // so if we don't report the error now, later failures will be - // very mysterious. - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - // Semantically the module version itself “exists” — we just don't - // have its source code. Remove the equivalence to os.ErrNotExist, - // and make the message more concise while we're at it. - err = fmt.Errorf("replacement directory %s does not exist", r.Path) - } else { - err = fmt.Errorf("replacement directory %s: %w", r.Path, err) - } - return dir, true, module.VersionError(mod, err) - } - return dir, true, nil - } - mod = r - } - - dir, err = modfetch.Download(ctx, mod) - return dir, false, err -} diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e75d901ec6..3927051015 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -8,17 +8,19 @@ import ( "context" "errors" "fmt" + "io/fs" "os" pathpkg "path" "path/filepath" + "sort" "strings" "sync" + "time" "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/trace" "golang.org/x/mod/module" @@ -106,138 +108,45 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed allowed = func(context.Context, module.Version) error { return nil } } - // Parse query to detect parse errors (and possibly handle query) - // before any network I/O. - badVersion := func(v string) (*modfetch.RevInfo, error) { - return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) - } - matchesMajor := func(v string) bool { - _, pathMajor, ok := module.SplitPathVersion(path) - if !ok { - return false - } - return module.CheckPathMajor(v, pathMajor) == nil - } - var ( - match = func(m module.Version) bool { return true } - - prefix string - preferOlder bool - mayUseLatest bool - preferIncompatible bool = strings.HasSuffix(current, "+incompatible") - ) - switch { - case query == "latest": - mayUseLatest = true - - case query == "upgrade": - mayUseLatest = true - - case query == "patch": - if current == "" { - mayUseLatest = true - } else { - prefix = semver.MajorMinor(current) - match = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) - } - } - - case strings.HasPrefix(query, "<="): - v := query[len("<="):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, "<"): - v := query[len("<"):] - if !semver.IsValid(v) { - return badVersion(v) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 - } - if !matchesMajor(v) { - preferIncompatible = true - } - - case strings.HasPrefix(query, ">="): - v := query[len(">="):] - if !semver.IsValid(v) { - return badVersion(v) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 + if path == Target.Path { + if query != "latest" { + return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } + return &modfetch.RevInfo{Version: Target.Version}, nil + } - case strings.HasPrefix(query, ">"): - v := query[len(">"):] - if !semver.IsValid(v) { - return badVersion(v) - } - if isSemverPrefix(v) { - // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). - return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) - } - match = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 - } - preferOlder = true - if !matchesMajor(v) { - preferIncompatible = true - } + if path == "std" || path == "cmd" { + return nil, fmt.Errorf("can't query specific version (%q) of standard-library module %q", query, path) + } - case semver.IsValid(query) && isSemverPrefix(query): - match = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) - } - prefix = query + "." - if !matchesMajor(query) { - preferIncompatible = true - } + repo, err := lookupRepo(proxy, path) + if err != nil { + return nil, err + } - default: - // Direct lookup of semantic version or commit identifier. - - // If the query is a valid semantic version and that version is replaced, - // use the replacement module without searching the proxy. - canonicalQuery := module.CanonicalVersion(query) - if canonicalQuery != "" { - m := module.Version{Path: path, Version: query} - if r := Replacement(m); r.Path != "" { - if err := allowed(ctx, m); errors.Is(err, ErrDisallowed) { - return nil, err - } - return &modfetch.RevInfo{Version: query}, nil - } - } + // Parse query to detect parse errors (and possibly handle query) + // before any network I/O. + qm, err := newQueryMatcher(path, query, current, allowed) + if (err == nil && qm.canStat) || err == errRevQuery { + // Direct lookup of a commit identifier or complete (non-prefix) semantic + // version. // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. - info, err := modfetch.Stat(proxy, path, query) + info, err := repo.Stat(query) if err != nil { queryErr := err // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. + canonicalQuery := module.CanonicalVersion(query) if canonicalQuery != "" && query != canonicalQuery { - info, err = modfetch.Stat(proxy, path, canonicalQuery) - if err != nil && !errors.Is(err, os.ErrNotExist) { + info, err = repo.Stat(canonicalQuery) + if err != nil && !errors.Is(err, fs.ErrNotExist) { return info, err } } @@ -249,38 +158,16 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, err } return info, nil - } - - if path == Target.Path { - if query != "latest" { - return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) - } - if err := allowed(ctx, Target); err != nil { - return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) - } - return &modfetch.RevInfo{Version: Target.Version}, nil - } - - if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") { - return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path) + } else if err != nil { + return nil, err } // Load versions and execute query. - repo, err := modfetch.Lookup(proxy, path) - if err != nil { - return nil, err - } - versions, err := repo.Versions(prefix) + versions, err := repo.Versions(qm.prefix) if err != nil { return nil, err } - matchAndAllowed := func(ctx context.Context, m module.Version) error { - if !match(m) { - return ErrDisallowed - } - return allowed(ctx, m) - } - releases, prereleases, err := filterVersions(ctx, path, versions, matchAndAllowed, preferIncompatible) + releases, prereleases, err := qm.filterVersions(ctx, versions) if err != nil { return nil, err } @@ -291,11 +178,30 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, err } - // For "upgrade" and "patch", make sure we don't accidentally downgrade - // from a newer prerelease or from a chronologically newer pseudoversion. - if current != "" && (query == "upgrade" || query == "patch") { + if (query == "upgrade" || query == "patch") && modfetch.IsPseudoVersion(current) && !rev.Time.IsZero() { + // Don't allow "upgrade" or "patch" to move from a pseudo-version + // to a chronologically older version or pseudo-version. + // + // If the current version is a pseudo-version from an untagged branch, it + // may be semantically lower than the "latest" release or the latest + // pseudo-version on the main branch. A user on such a version is unlikely + // to intend to “upgrade” to a version that already existed at that point + // in time. + // + // We do this only if the current version is a pseudo-version: if the + // version is tagged, the author of the dependency module has given us + // explicit information about their intended precedence of this version + // relative to other versions, and we shouldn't contradict that + // information. (For example, v1.0.1 might be a backport of a fix already + // incorporated into v1.1.0, in which case v1.0.1 would be chronologically + // newer but v1.1.0 is still an “upgrade”; or v1.0.2 might be a revert of + // an unsuccessful fix in v1.0.1, in which case the v1.0.2 commit may be + // older than the v1.0.1 commit despite the tag itself being newer.) currentTime, err := modfetch.PseudoVersionTime(current) - if semver.Compare(rev.Version, current) < 0 || (err == nil && rev.Time.Before(currentTime)) { + if err == nil && rev.Time.Before(currentTime) { + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } return repo.Stat(current) } } @@ -303,7 +209,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return rev, nil } - if preferOlder { + if qm.preferLower { if len(releases) > 0 { return lookup(releases[0]) } @@ -319,20 +225,25 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } } - if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo - // if it is allowed. + if qm.mayUseLatest { latest, err := repo.Latest() if err == nil { - m := module.Version{Path: path, Version: latest.Version} - if err := allowed(ctx, m); !errors.Is(err, ErrDisallowed) { + if qm.allowsVersion(ctx, latest.Version) { return lookup(latest.Version) } - } else if !errors.Is(err, os.ErrNotExist) { + } else if !errors.Is(err, fs.ErrNotExist) { return nil, err } } + if (query == "upgrade" || query == "patch") && current != "" { + // "upgrade" and "patch" may stay on the current version if allowed. + if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) { + return nil, err + } + return lookup(current) + } + return nil, &NoMatchingVersionError{query: query, current: current} } @@ -370,10 +281,151 @@ func isSemverPrefix(v string) bool { return true } -// matchSemverPrefix reports whether the shortened semantic version p -// matches the full-width (non-shortened) semantic version v. -func matchSemverPrefix(p, v string) bool { - return len(v) > len(p) && v[len(p)] == '.' && v[:len(p)] == p && semver.Prerelease(v) == "" +type queryMatcher struct { + path string + prefix string + filter func(version string) bool + allowed AllowedFunc + canStat bool // if true, the query can be resolved by repo.Stat + preferLower bool // if true, choose the lowest matching version + mayUseLatest bool + preferIncompatible bool +} + +var errRevQuery = errors.New("query refers to a non-semver revision") + +// newQueryMatcher returns a new queryMatcher that matches the versions +// specified by the given query on the module with the given path. +// +// If the query can only be resolved by statting a non-SemVer revision, +// newQueryMatcher returns errRevQuery. +func newQueryMatcher(path string, query, current string, allowed AllowedFunc) (*queryMatcher, error) { + badVersion := func(v string) (*queryMatcher, error) { + return nil, fmt.Errorf("invalid semantic version %q in range %q", v, query) + } + + matchesMajor := func(v string) bool { + _, pathMajor, ok := module.SplitPathVersion(path) + if !ok { + return false + } + return module.CheckPathMajor(v, pathMajor) == nil + } + + qm := &queryMatcher{ + path: path, + allowed: allowed, + preferIncompatible: strings.HasSuffix(current, "+incompatible"), + } + + switch { + case query == "latest": + qm.mayUseLatest = true + + case query == "upgrade": + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case query == "patch": + if current == "" { + qm.mayUseLatest = true + } else { + qm.mayUseLatest = modfetch.IsPseudoVersion(current) + qm.prefix = semver.MajorMinor(current) + "." + qm.filter = func(mv string) bool { return semver.Compare(mv, current) >= 0 } + } + + case strings.HasPrefix(query, "<="): + v := query[len("<="):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) <= 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, "<"): + v := query[len("<"):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) < 0 } + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">="): + v := query[len(">="):] + if !semver.IsValid(v) { + return badVersion(v) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) >= 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case strings.HasPrefix(query, ">"): + v := query[len(">"):] + if !semver.IsValid(v) { + return badVersion(v) + } + if isSemverPrefix(v) { + // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). + return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) + } + qm.filter = func(mv string) bool { return semver.Compare(mv, v) > 0 } + qm.preferLower = true + if !matchesMajor(v) { + qm.preferIncompatible = true + } + + case semver.IsValid(query): + if isSemverPrefix(query) { + qm.prefix = query + "." + // Do not allow the query "v1.2" to match versions lower than "v1.2.0", + // such as prereleases for that version. (https://golang.org/issue/31972) + qm.filter = func(mv string) bool { return semver.Compare(mv, query) >= 0 } + } else { + qm.canStat = true + qm.filter = func(mv string) bool { return semver.Compare(mv, query) == 0 } + qm.prefix = semver.Canonical(query) + } + if !matchesMajor(query) { + qm.preferIncompatible = true + } + + default: + return nil, errRevQuery + } + + return qm, nil +} + +// allowsVersion reports whether version v is allowed by the prefix, filter, and +// AllowedFunc of qm. +func (qm *queryMatcher) allowsVersion(ctx context.Context, v string) bool { + if qm.prefix != "" && !strings.HasPrefix(v, qm.prefix) { + return false + } + if qm.filter != nil && !qm.filter(v) { + return false + } + if qm.allowed != nil { + if err := qm.allowed(ctx, module.Version{Path: qm.path, Version: v}); errors.Is(err, ErrDisallowed) { + return false + } + } + return true } // filterVersions classifies versions into releases and pre-releases, filtering @@ -384,14 +436,32 @@ func matchSemverPrefix(p, v string) bool { // // If the allowed predicate returns an error not equivalent to ErrDisallowed, // filterVersions returns that error. -func filterVersions(ctx context.Context, path string, versions []string, allowed AllowedFunc, preferIncompatible bool) (releases, prereleases []string, err error) { +func (qm *queryMatcher) filterVersions(ctx context.Context, versions []string) (releases, prereleases []string, err error) { + needIncompatible := qm.preferIncompatible + var lastCompatible string for _, v := range versions { - if err := allowed(ctx, module.Version{Path: path, Version: v}); errors.Is(err, ErrDisallowed) { + if !qm.allowsVersion(ctx, v) { continue } - if !preferIncompatible { + if !needIncompatible { + // We're not yet sure whether we need to include +incomptaible versions. + // Keep track of the last compatible version we've seen, and use the + // presence (or absence) of a go.mod file in that version to decide: a + // go.mod file implies that the module author is supporting modules at a + // compatible version (and we should ignore +incompatible versions unless + // requested explicitly), while a lack of go.mod file implies the + // potential for legacy (pre-modules) versioning without semantic import + // paths (and thus *with* +incompatible versions). + // + // This isn't strictly accurate if the latest compatible version has been + // replaced by a local file path, because we do not allow file-path + // replacements without a go.mod file: the user would have needed to add + // one. However, replacing the last compatible version while + // simultaneously expecting to upgrade implicitly to a +incompatible + // version seems like an extreme enough corner case to ignore for now. + if !strings.HasSuffix(v, "+incompatible") { lastCompatible = v } else if lastCompatible != "" { @@ -399,19 +469,22 @@ func filterVersions(ctx context.Context, path string, versions []string, allowed // ignore any version with a higher (+incompatible) major version. (See // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - - ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: qm.path, Version: lastCompatible}) if err != nil { return nil, nil, err } if ok { + // The last compatible version has a go.mod file, so that's the + // highest version we're willing to consider. Don't bother even + // looking at higher versions, because they're all +incompatible from + // here onward. break } // No acceptable compatible release has a go.mod file, so the versioning // for the module might not be module-aware, and we should respect // legacy major-version tags. - preferIncompatible = true + needIncompatible = true } } @@ -444,7 +517,7 @@ type QueryResult struct { // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFunc) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, current func(string) string, allowed AllowedFunc) ([]QueryResult, error) { ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) defer span.Done() @@ -519,14 +592,15 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFun ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) defer span.Done() - current := findCurrentVersion(path) + pathCurrent := current(path) r.Mod.Path = path - r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(ctx, r.Mod) + needSum := true + root, isLocal, err := fetch(ctx, r.Mod, needSum) if err != nil { return r, err } @@ -577,15 +651,6 @@ func modulePrefixesExcludingTarget(path string) []string { return prefixes } -func findCurrentVersion(path string) string { - for _, m := range buildList { - if m.Path == path { - return m.Version - } - } - return "" -} - type prefixResult struct { QueryResult err error @@ -638,7 +703,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod noVersion = rErr } default: - if errors.Is(rErr, os.ErrNotExist) { + if errors.Is(rErr, fs.ErrNotExist) { if notExistErr == nil { notExistErr = rErr } @@ -681,7 +746,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod // A NoMatchingVersionError indicates that Query found a module at the requested // path, but not at any versions satisfying the query string and allow-function. // -// NOTE: NoMatchingVersionError MUST NOT implement Is(os.ErrNotExist). +// NOTE: NoMatchingVersionError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -702,7 +767,7 @@ func (e *NoMatchingVersionError) Error() string { // module at the requested version, but that module did not contain any packages // matching the requested pattern. // -// NOTE: PackageNotInModuleError MUST NOT implement Is(os.ErrNotExist). +// NOTE: PackageNotInModuleError MUST NOT implement Is(fs.ErrNotExist). // // If the module came from a proxy, that proxy had to return a successful status // code for the versions it knows about, and thus did not have the opportunity @@ -752,7 +817,8 @@ func (e *PackageNotInModuleError) ImportPath() string { // ModuleHasRootPackage returns whether module m contains a package m.Path. func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { - root, isLocal, err := fetch(ctx, m) + needSum := false + root, isLocal, err := fetch(ctx, m, needSum) if err != nil { return false, err } @@ -761,10 +827,165 @@ func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { } func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { - root, _, err := fetch(ctx, m) + needSum := false + root, _, err := fetch(ctx, m, needSum) if err != nil { return false, err } fi, err := os.Stat(filepath.Join(root, "go.mod")) return err == nil && !fi.IsDir(), nil } + +// A versionRepo is a subset of modfetch.Repo that can report information about +// available versions, but cannot fetch specific source files. +type versionRepo interface { + ModulePath() string + Versions(prefix string) ([]string, error) + Stat(rev string) (*modfetch.RevInfo, error) + Latest() (*modfetch.RevInfo, error) +} + +var _ versionRepo = modfetch.Repo(nil) + +func lookupRepo(proxy, path string) (repo versionRepo, err error) { + err = module.CheckPath(path) + if err == nil { + repo = modfetch.Lookup(proxy, path) + } else { + repo = emptyRepo{path: path, err: err} + } + + if index == nil { + return repo, err + } + if _, ok := index.highestReplaced[path]; !ok { + return repo, err + } + + return &replacementRepo{repo: repo}, nil +} + +// An emptyRepo is a versionRepo that contains no versions. +type emptyRepo struct { + path string + err error +} + +var _ versionRepo = emptyRepo{} + +func (er emptyRepo) ModulePath() string { return er.path } +func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil } +func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err } +func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err } + +// A replacementRepo augments a versionRepo to include the replacement versions +// (if any) found in the main module's go.mod file. +// +// A replacementRepo suppresses "not found" errors for otherwise-nonexistent +// modules, so a replacementRepo should only be constructed for a module that +// actually has one or more valid replacements. +type replacementRepo struct { + repo versionRepo +} + +var _ versionRepo = (*replacementRepo)(nil) + +func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() } + +// Versions returns the versions from rr.repo augmented with any matching +// replacement versions. +func (rr *replacementRepo) Versions(prefix string) ([]string, error) { + repoVersions, err := rr.repo.Versions(prefix) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + versions := repoVersions + if index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !modfetch.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } + } + } + + if len(versions) == len(repoVersions) { // No replacement versions added. + return versions, nil + } + + sort.Slice(versions, func(i, j int) bool { + return semver.Compare(versions[i], versions[j]) < 0 + }) + uniq := versions[:1] + for _, v := range versions { + if v != uniq[len(uniq)-1] { + uniq = append(uniq, v) + } + } + return uniq, nil +} + +func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { + info, err := rr.repo.Stat(rev) + if err == nil || index == nil || len(index.replace) == 0 { + return info, err + } + + v := module.CanonicalVersion(rev) + if v != rev { + // The replacements in the go.mod file list only canonical semantic versions, + // so a non-canonical version can't possibly have a replacement. + return info, err + } + + path := rr.ModulePath() + _, pathMajor, ok := module.SplitPathVersion(path) + if ok && pathMajor == "" { + if err := module.CheckPathMajor(v, pathMajor); err != nil && semver.Build(v) == "" { + v += "+incompatible" + } + } + + if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + return info, err + } + return rr.replacementStat(v) +} + +func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { + info, err := rr.repo.Latest() + + if index != nil { + path := rr.ModulePath() + if v, ok := index.highestReplaced[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = modfetch.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = modfetch.PseudoVersion("v0", "", time.Time{}, "000000000000") + } + } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } + } + } + + return info, err +} + +func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) { + rev := &modfetch.RevInfo{Version: v} + if modfetch.IsPseudoVersion(v) { + rev.Time, _ = modfetch.PseudoVersionTime(v) + rev.Short, _ = modfetch.PseudoVersionRev(v) + } + return rev, nil +} diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 351826f2ab..777a56b977 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -45,7 +45,7 @@ var ( queryRepoV3 = queryRepo + "/v3" // Empty version list (no semver tags), not actually empty. - emptyRepo = "vcs-test.golang.org/git/emptytest.git" + emptyRepoPath = "vcs-test.golang.org/git/emptytest.git" ) var queryTests = []struct { @@ -121,14 +121,14 @@ var queryTests = []struct { {path: queryRepo, query: "upgrade", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", current: "v0.0.0-20190513201126-42abcb6df8ee", vers: "v0.0.0-20190513201126-42abcb6df8ee"}, {path: queryRepo, query: "upgrade", allow: "NOMATCH", err: `no matching versions for query "upgrade"`}, - {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `no matching versions for query "upgrade" (current version is v1.9.9)`}, + {path: queryRepo, query: "upgrade", current: "v1.9.9", allow: "NOMATCH", err: `vcs-test.golang.org/git/querytest.git@v1.9.9: disallowed module version`}, {path: queryRepo, query: "upgrade", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: "patch", current: "", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v0.1.0", vers: "v0.1.2"}, {path: queryRepo, query: "patch", current: "v1.9.0", vers: "v1.9.9"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre1", vers: "v1.9.10-pre1"}, {path: queryRepo, query: "patch", current: "v1.9.10-pre2+metadata", vers: "v1.9.10-pre2.0.20190513201126-42abcb6df8ee"}, - {path: queryRepo, query: "patch", current: "v1.99.99", err: `no matching versions for query "patch" (current version is v1.99.99)`}, + {path: queryRepo, query: "patch", current: "v1.99.99", err: `vcs-test.golang.org/git/querytest.git@v1.99.99: invalid version: unknown revision v1.99.99`}, {path: queryRepo, query: ">v1.9.9", vers: "v1.9.10-pre1"}, {path: queryRepo, query: ">v1.10.0", err: `no matching versions for query ">v1.10.0"`}, {path: queryRepo, query: ">=v1.10.0", err: `no matching versions for query ">=v1.10.0"`}, @@ -171,9 +171,9 @@ var queryTests = []struct { // That should prevent us from resolving any version for the /v3 path. {path: queryRepoV3, query: "latest", err: `no matching versions for query "latest"`}, - {path: emptyRepo, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, - {path: emptyRepo, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, - {path: emptyRepo, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, + {path: emptyRepoPath, query: "latest", vers: "v0.0.0-20180704023549-7bb914627242"}, + {path: emptyRepoPath, query: ">v0.0.0", err: `no matching versions for query ">v0.0.0"`}, + {path: emptyRepoPath, query: "<v10.0.0", err: `no matching versions for query "<v10.0.0"`}, } func TestQuery(t *testing.T) { @@ -189,7 +189,7 @@ func TestQuery(t *testing.T) { } allowed := func(ctx context.Context, m module.Version) error { if ok, _ := path.Match(allow, m.Version); !ok { - return ErrDisallowed + return module.VersionError(m, ErrDisallowed) } return nil } @@ -200,17 +200,17 @@ func TestQuery(t *testing.T) { info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { - t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want error %q", tt.path, tt.query, tt.current, allow, info.Version, tt.err) } else if err.Error() != tt.err { - t.Errorf("Query(%q, %q, %v): %v, want error %q", tt.path, tt.query, allow, err, tt.err) + t.Errorf("Query(_, %q, %q, %q, %v): %v\nwant error %q", tt.path, tt.query, tt.current, allow, err, tt.err) } return } if err != nil { - t.Fatalf("Query(%q, %q, %v): %v", tt.path, tt.query, allow, err) + t.Fatalf("Query(_, %q, %q, %q, %v): %v\nwant %v", tt.path, tt.query, tt.current, allow, err, tt.vers) } if info.Version != tt.vers { - t.Errorf("Query(%q, %q, %v) = %v, want %v", tt.path, tt.query, allow, info.Version, tt.vers) + t.Errorf("Query(_, %q, %q, %q, %v) = %v, want %v", tt.path, tt.query, tt.current, allow, info.Version, tt.vers) } }) } diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index a9bee0af4e..f6d6f5f764 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -7,11 +7,13 @@ package modload import ( "context" "fmt" + "io/fs" "os" "path/filepath" "strings" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/search" @@ -53,7 +55,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f walkPkgs := func(root, importPathRoot string, prune pruning) { root = filepath.Clean(root) - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { m.AddError(err) return nil @@ -84,8 +86,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -154,7 +156,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f isLocal = true } else { var err error - root, isLocal, err = fetch(ctx, mod) + needSum := true + root, isLocal, err = fetch(ctx, mod, needSum) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/modload/stat_openfile.go b/src/cmd/go/internal/modload/stat_openfile.go index 931aaf1577..5842b858f0 100644 --- a/src/cmd/go/internal/modload/stat_openfile.go +++ b/src/cmd/go/internal/modload/stat_openfile.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix js,wasm plan9 +// +build js,wasm plan9 // On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions // are checked by the server and group information is not known to the client, @@ -13,12 +13,13 @@ package modload import ( + "io/fs" "os" ) // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(path string, _ os.FileInfo) bool { +func hasWritePerm(path string, _ fs.FileInfo) bool { if f, err := os.OpenFile(path, os.O_WRONLY, 0); err == nil { f.Close() return true diff --git a/src/cmd/go/internal/modload/stat_unix.go b/src/cmd/go/internal/modload/stat_unix.go index ea3b801f2c..f49278ec3a 100644 --- a/src/cmd/go/internal/modload/stat_unix.go +++ b/src/cmd/go/internal/modload/stat_unix.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package modload import ( + "io/fs" "os" "syscall" ) @@ -17,7 +18,7 @@ import ( // Although the root user on most Unix systems can write to files even without // permission, hasWritePerm reports false if no appropriate permission bit is // set even if the current user is root. -func hasWritePerm(path string, fi os.FileInfo) bool { +func hasWritePerm(path string, fi fs.FileInfo) bool { if os.Getuid() == 0 { // The root user can access any file, but we still want to default to // read-only mode if the go.mod file is marked as globally non-writable. diff --git a/src/cmd/go/internal/modload/stat_windows.go b/src/cmd/go/internal/modload/stat_windows.go index d7826cfc6b..0ac2391347 100644 --- a/src/cmd/go/internal/modload/stat_windows.go +++ b/src/cmd/go/internal/modload/stat_windows.go @@ -6,13 +6,11 @@ package modload -import ( - "os" -) +import "io/fs" // hasWritePerm reports whether the current user has permission to write to the // file with the given info. -func hasWritePerm(_ string, fi os.FileInfo) bool { +func hasWritePerm(_ string, fi fs.FileInfo) bool { // Windows has a read-only attribute independent of ACLs, so use that to // determine whether the file is intended to be overwritten. // diff --git a/src/cmd/go/internal/modload/testgo.go b/src/cmd/go/internal/modload/testgo.go deleted file mode 100644 index 6b34f5be39..0000000000 --- a/src/cmd/go/internal/modload/testgo.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build testgo - -package modload - -func init() { - printStackInDie = true -} diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 9f34b829fc..ab29d4d014 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -7,8 +7,8 @@ package modload import ( "errors" "fmt" + "io/fs" "io/ioutil" - "os" "path/filepath" "strings" "sync" @@ -42,7 +42,7 @@ func readVendorList() { vendorMeta = make(map[module.Version]vendorMetadata) data, err := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) if err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) } return diff --git a/src/cmd/go/internal/renameio/renameio.go b/src/cmd/go/internal/renameio/renameio.go index d573cc690d..60a7138a76 100644 --- a/src/cmd/go/internal/renameio/renameio.go +++ b/src/cmd/go/internal/renameio/renameio.go @@ -8,6 +8,7 @@ package renameio import ( "bytes" "io" + "io/fs" "math/rand" "os" "path/filepath" @@ -29,13 +30,13 @@ func Pattern(filename string) string { // final name. // // That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { +func WriteFile(filename string, data []byte, perm fs.FileMode) (err error) { return WriteToFile(filename, bytes.NewReader(data), perm) } // WriteToFile is a variant of WriteFile that accepts the data as an io.Reader // instead of a slice. -func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { +func WriteToFile(filename string, data io.Reader, perm fs.FileMode) (err error) { f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) if err != nil { return err @@ -80,7 +81,7 @@ func ReadFile(filename string) ([]byte, error) { } // tempFile creates a new temporary file with given permission bits. -func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { +func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) { for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) diff --git a/src/cmd/go/internal/renameio/renameio_test.go b/src/cmd/go/internal/renameio/renameio_test.go index df8ddabdb8..e6d2025a0e 100644 --- a/src/cmd/go/internal/renameio/renameio_test.go +++ b/src/cmd/go/internal/renameio/renameio_test.go @@ -144,10 +144,12 @@ func TestConcurrentReadsAndWrites(t *testing.T) { // As long as those are the only errors and *some* of the reads succeed, we're happy. minReadSuccesses = attempts / 4 - case "darwin": - // The filesystem on macOS 10.14 occasionally fails with "no such file or - // directory" errors. See https://golang.org/issue/33041. The flake rate is - // fairly low, so ensure that at least 75% of attempts succeed. + case "darwin", "ios": + // The filesystem on certain versions of macOS (10.14) and iOS (affected + // versions TBD) occasionally fail with "no such file or directory" errors. + // See https://golang.org/issue/33041 and https://golang.org/issue/42066. + // The flake rate is fairly low, so ensure that at least 75% of attempts + // succeed. minReadSuccesses = attempts - (attempts / 4) } diff --git a/src/cmd/go/internal/renameio/umask_test.go b/src/cmd/go/internal/renameio/umask_test.go index d75d67c9a9..19e217c548 100644 --- a/src/cmd/go/internal/renameio/umask_test.go +++ b/src/cmd/go/internal/renameio/umask_test.go @@ -7,6 +7,7 @@ package renameio import ( + "io/fs" "io/ioutil" "os" "path/filepath" @@ -36,7 +37,7 @@ func TestWriteFileModeAppliesUmask(t *testing.T) { t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) } - if fi.Mode()&os.ModePerm != 0640 { - t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640) + if fi.Mode()&fs.ModePerm != 0640 { + t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&fs.ModePerm, 0640) } } diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index 4efef24152..57cbb282a8 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -7,8 +7,10 @@ package search import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "fmt" "go/build" + "io/fs" "os" "path" "path/filepath" @@ -127,7 +129,7 @@ func (m *Match) MatchPackages() { if m.pattern == "cmd" { root += "cmd" + string(filepath.Separator) } - err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -153,8 +155,8 @@ func (m *Match) MatchPackages() { } if !fi.IsDir() { - if fi.Mode()&os.ModeSymlink != 0 && want { - if target, err := os.Stat(path); err == nil && target.IsDir() { + if fi.Mode()&fs.ModeSymlink != 0 && want { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) } } @@ -263,7 +265,7 @@ func (m *Match) MatchDirs() { } } - err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + err := fsys.Walk(dir, func(path string, fi fs.FileInfo, err error) error { if err != nil { return err // Likely a permission error, which could interfere with matching. } @@ -272,7 +274,7 @@ func (m *Match) MatchDirs() { } top := false if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, + // Walk starts at dir and recurses. For the recursive case, // the path is the result of filepath.Join, which calls filepath.Clean. // The initial case is not Cleaned, though, so we do this explicitly. // diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 51d333d866..00da9770df 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -12,6 +12,7 @@ import ( "fmt" "go/build" "io" + "io/fs" "io/ioutil" "os" "os/exec" @@ -1598,7 +1599,7 @@ func hashStat(name string) cache.ActionID { return h.Sum() } -func hashWriteStat(h io.Writer, info os.FileInfo) { +func hashWriteStat(h io.Writer, info fs.FileInfo) { fmt.Fprintf(h, "stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) } diff --git a/src/cmd/go/internal/vcs/vcs.go b/src/cmd/go/internal/vcs/vcs.go index 90bf10244d..7812afd488 100644 --- a/src/cmd/go/internal/vcs/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -10,6 +10,7 @@ import ( "fmt" "internal/lazyregexp" "internal/singleflight" + "io/fs" "log" urlpkg "net/url" "os" @@ -404,9 +405,9 @@ func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([ if len(args) >= 2 && args[0] == "-go-internal-mkdir" { var err error if filepath.IsAbs(args[1]) { - err = os.Mkdir(args[1], os.ModePerm) + err = os.Mkdir(args[1], fs.ModePerm) } else { - err = os.Mkdir(filepath.Join(dir, args[1]), os.ModePerm) + err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) } if err != nil { return nil, err diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index 5aa0f8e7ed..44ac24c62d 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -10,6 +10,7 @@ import ( "context" "encoding/binary" "fmt" + "io/fs" "os" "path/filepath" "runtime" @@ -87,8 +88,8 @@ func runVersion(ctx context.Context, cmd *base.Command, args []string) { // scanDir scans a directory for executables to run scanFile on. func scanDir(dir string) { - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if info.Mode().IsRegular() || info.Mode()&fs.ModeSymlink != 0 { scanFile(path, info, *versionV) } return nil @@ -96,7 +97,7 @@ func scanDir(dir string) { } // isExe reports whether the file should be considered executable. -func isExe(file string, info os.FileInfo) bool { +func isExe(file string, info fs.FileInfo) bool { if runtime.GOOS == "windows" { return strings.HasSuffix(strings.ToLower(file), ".exe") } @@ -107,8 +108,8 @@ func isExe(file string, info os.FileInfo) bool { // If mustPrint is true, scanFile will report any error reading file. // Otherwise (mustPrint is false, because scanFile is being called // by scanDir) scanFile prints nothing for non-Go executables. -func scanFile(file string, info os.FileInfo, mustPrint bool) { - if info.Mode()&os.ModeSymlink != 0 { +func scanFile(file string, info fs.FileInfo, mustPrint bool) { + if info.Mode()&fs.ModeSymlink != 0 { // Accept file symlinks only. i, err := os.Stat(file) if err != nil || !i.Mode().IsRegular() { diff --git a/src/cmd/go/internal/web/api.go b/src/cmd/go/internal/web/api.go index 570818843b..9053b16b62 100644 --- a/src/cmd/go/internal/web/api.go +++ b/src/cmd/go/internal/web/api.go @@ -13,9 +13,8 @@ import ( "bytes" "fmt" "io" - "io/ioutil" + "io/fs" "net/url" - "os" "strings" "unicode" "unicode/utf8" @@ -56,7 +55,7 @@ func (e *HTTPError) Error() string { } if err := e.Err; err != nil { - if pErr, ok := e.Err.(*os.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { + if pErr, ok := e.Err.(*fs.PathError); ok && strings.HasSuffix(e.URL, pErr.Path) { // Remove the redundant copy of the path. err = pErr.Err } @@ -67,7 +66,7 @@ func (e *HTTPError) Error() string { } func (e *HTTPError) Is(target error) bool { - return target == os.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) + return target == fs.ErrNotExist && (e.StatusCode == 404 || e.StatusCode == 410) } func (e *HTTPError) Unwrap() error { @@ -87,7 +86,7 @@ func GetBytes(u *url.URL) ([]byte, error) { if err := resp.Err(); err != nil { return nil, err } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading %s: %v", u.Redacted(), err) } @@ -130,7 +129,7 @@ func (r *Response) formatErrorDetail() string { } // Ensure that r.errorDetail has been populated. - _, _ = io.Copy(ioutil.Discard, r.Body) + _, _ = io.Copy(io.Discard, r.Body) s := r.errorDetail.buf.String() if !utf8.ValidString(s) { diff --git a/src/cmd/go/internal/web/file_test.go b/src/cmd/go/internal/web/file_test.go index 6339469045..a1bb080e07 100644 --- a/src/cmd/go/internal/web/file_test.go +++ b/src/cmd/go/internal/web/file_test.go @@ -6,6 +6,7 @@ package web import ( "errors" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -54,7 +55,7 @@ func TestGetNonexistentFile(t *testing.T) { } b, err := GetBytes(u) - if !errors.Is(err, os.ErrNotExist) { - t.Fatalf("GetBytes(%v) = %q, %v; want _, os.ErrNotExist", u, b, err) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("GetBytes(%v) = %q, %v; want _, fs.ErrNotExist", u, b, err) } } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 86423f118c..3531612dc6 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modfetch" "cmd/go/internal/modload" @@ -277,6 +278,8 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") + cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") + // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") @@ -738,7 +741,7 @@ func installOutsideModule(ctx context.Context, args []string) { // Don't check for retractions if a specific revision is requested. allowed = nil } - qrs, err := modload.QueryPattern(ctx, patterns[0], version, allowed) + qrs, err := modload.QueryPattern(ctx, patterns[0], version, modload.Selected, allowed) if err != nil { base.Fatalf("go install %s: %v", args[0], err) } diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go index afed0fba72..e941729734 100644 --- a/src/cmd/go/internal/work/build_test.go +++ b/src/cmd/go/internal/work/build_test.go @@ -7,6 +7,7 @@ package work import ( "bytes" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -221,10 +222,8 @@ func pkgImportPath(pkgpath string) *load.Package { // See https://golang.org/issue/18878. func TestRespectSetgidDir(t *testing.T) { switch runtime.GOOS { - case "darwin", "ios": - if runtime.GOARCH == "arm64" { - t.Skip("can't set SetGID bit with chmod on iOS") - } + case "ios": + t.Skip("can't set SetGID bit with chmod on iOS") case "windows", "plan9": t.Skip("chown/chmod setgid are not supported on Windows or Plan 9") } @@ -255,7 +254,7 @@ func TestRespectSetgidDir(t *testing.T) { } // Change setgiddir's permissions to include the SetGID bit. - if err := os.Chmod(setgiddir, 0755|os.ModeSetgid); err != nil { + if err := os.Chmod(setgiddir, 0755|fs.ModeSetgid); err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index a3c9b1a2c1..5cd3124e54 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -713,6 +713,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { return err } a.Package.Export = c.OutputFile(outputID) + a.Package.BuildID = a.buildID } } } diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 51fc2b588d..24e309c657 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -14,6 +14,7 @@ import ( "fmt" "internal/lazyregexp" "io" + "io/fs" "io/ioutil" "log" "math/rand" @@ -271,7 +272,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) @@ -432,6 +433,7 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { need &^= needBuild if b.NeedExport { p.Export = a.built + p.BuildID = a.buildID } if need&needCompiledGoFiles != 0 { if err := b.loadCachedSrcFiles(a); err == nil { @@ -922,12 +924,13 @@ func (b *Builder) loadCachedSrcFiles(a *Action) error { // vetConfig is the configuration passed to vet describing a single package. type vetConfig struct { - ID string // package ID (example: "fmt [fmt.test]") - Compiler string // compiler name (gc, gccgo) - Dir string // directory containing package - ImportPath string // canonical import path ("package path") - GoFiles []string // absolute paths to package source files - NonGoFiles []string // absolute paths to package non-Go files + ID string // package ID (example: "fmt [fmt.test]") + Compiler string // compiler name (gc, gccgo) + Dir string // directory containing package + ImportPath string // canonical import path ("package path") + GoFiles []string // absolute paths to package source files + NonGoFiles []string // absolute paths to package non-Go files + IgnoredFiles []string // absolute paths to ignored source files ImportMap map[string]string // map import path in source code to package path PackageFile map[string]string // map package path to .a file with export data @@ -951,20 +954,23 @@ func buildVetConfig(a *Action, srcfiles []string) { } } + ignored := str.StringList(a.Package.IgnoredGoFiles, a.Package.IgnoredOtherFiles) + // Pass list of absolute paths to vet, // so that vet's error messages will use absolute paths, // so that we can reformat them relative to the directory // in which the go command is invoked. vcfg := &vetConfig{ - ID: a.Package.ImportPath, - Compiler: cfg.BuildToolchainName, - Dir: a.Package.Dir, - GoFiles: mkAbsFiles(a.Package.Dir, gofiles), - NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), - ImportPath: a.Package.ImportPath, - ImportMap: make(map[string]string), - PackageFile: make(map[string]string), - Standard: make(map[string]bool), + ID: a.Package.ImportPath, + Compiler: cfg.BuildToolchainName, + Dir: a.Package.Dir, + GoFiles: mkAbsFiles(a.Package.Dir, gofiles), + NonGoFiles: mkAbsFiles(a.Package.Dir, nongofiles), + IgnoredFiles: mkAbsFiles(a.Package.Dir, ignored), + ImportPath: a.Package.ImportPath, + ImportMap: make(map[string]string), + PackageFile: make(map[string]string), + Standard: make(map[string]bool), } a.vetCfg = vcfg for i, raw := range a.Package.Internal.RawImports { @@ -1052,17 +1058,28 @@ func (b *Builder) vet(ctx context.Context, a *Action) error { // This is OK as long as the packages that are farther down the // dependency tree turn on *more* analysis, as here. // (The unsafeptr check does not write any facts for use by - // later vet runs.) + // later vet runs, nor does unreachable.) if a.Package.Goroot && !VetExplicit && VetTool == "" { + // Turn off -unsafeptr checks. + // There's too much unsafe.Pointer code + // that vet doesn't like in low-level packages + // like runtime, sync, and reflect. // Note that $GOROOT/src/buildall.bash // does the same for the misc-compile trybots // and should be updated if these flags are // changed here. - // - // There's too much unsafe.Pointer code - // that vet doesn't like in low-level packages - // like runtime, sync, and reflect. vetFlags = []string{"-unsafeptr=false"} + + // Also turn off -unreachable checks during go test. + // During testing it is very common to make changes + // like hard-coded forced returns or panics that make + // code unreachable. It's unreasonable to insist on files + // not having any unreachable code during "go test". + // (buildall.bash still runs with -unreachable enabled + // for the overall whole-tree scan.) + if cfg.CmdName == "test" { + vetFlags = append(vetFlags, "-unreachable=false") + } } // Note: We could decide that vet should compute export data for @@ -1175,7 +1192,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { fmt.Fprintf(h, "linkflags %q\n", p.Internal.Ldflags) } - // GO386, GOARM, GOMIPS, etc. + // GOARM, GOMIPS, etc. key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) @@ -1545,7 +1562,7 @@ func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { return err } - perm := os.FileMode(0666) + perm := fs.FileMode(0666) if a1.Mode == "link" { switch cfg.BuildBuildmode { case "c-archive", "c-shared", "plugin": @@ -1594,7 +1611,7 @@ func (b *Builder) cleanup(a *Action) { } // moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN { b.Showcmd("", "mv %s %s", src, dst) return nil @@ -1620,7 +1637,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) // we have to copy the file to retain the correct permissions. // https://golang.org/issue/18878 if fi, err := os.Stat(filepath.Dir(dst)); err == nil { - if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { + if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { return b.copyFile(dst, src, perm, force) } } @@ -1655,7 +1672,7 @@ func (b *Builder) moveOrCopyFile(dst, src string, perm os.FileMode, force bool) } // copyFile is like 'cp src dst'. -func (b *Builder) copyFile(dst, src string, perm os.FileMode, force bool) error { +func (b *Builder) copyFile(dst, src string, perm fs.FileMode, force bool) error { if cfg.BuildN || cfg.BuildX { b.Showcmd("", "cp %s %s", src, dst) if cfg.BuildN { @@ -1984,6 +2001,13 @@ func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interfa defer cleanup() cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) + + // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. + // It doesn't really matter if -toolexec isn't being used. + if a != nil && a.Package != nil { + cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.ImportPath) + } + cmd.Env = append(cmd.Env, env...) start := time.Now() err := cmd.Run() @@ -2199,6 +2223,8 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s // when -trimpath is enabled. if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { if cfg.BuildTrimpath { + // TODO(#39958): handle overlays + // Keep in sync with Action.trimpath. // The trimmed paths are a little different, but we need to trim in the // same situations. diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index d76574932e..0c4a7fa6e3 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -18,6 +18,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/objabi" @@ -88,7 +89,11 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) if p.Standard { switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": extFiles++ } } @@ -145,10 +150,26 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(a, p.Dir, nil, args...) + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) + } + + output, err = b.runOut(a, base.Cwd, nil, args...) return ofile, output, err } @@ -237,13 +258,26 @@ func (a *Action) trimpath() string { } rewrite := objdir + "=>" - // For "go build -trimpath", rewrite package source directory - // to a file system-independent path (just the import path). + rewriteDir := a.Package.Dir if cfg.BuildTrimpath { if m := a.Package.Module; m != nil && m.Version != "" { - rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) } else { - rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath + rewriteDir = a.Package.ImportPath + } + rewrite += ";" + a.Package.Dir + "=>" + rewriteDir + } + + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + overlayPath, ok := fsys.OverlayPath(filepath.Join(a.Package.Dir, filename)) + if !ok { + continue + } + rewrite += ";" + overlayPath + "=>" + filepath.Join(rewriteDir, filename) } } @@ -262,14 +296,20 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } - if p.ImportPath == "runtime" && objabi.Regabi_enabled != 0 { - // In order to make it easier to port runtime assembly - // to the register ABI, we introduce a macro - // indicating the experiment is enabled. - // - // TODO(austin): Remove this once we commit to the - // register ABI (#40724). - args = append(args, "-D=GOEXPERIMENT_REGABI=1") + if objabi.IsRuntimePackagePath(pkgpath) { + args = append(args, "-compiling-runtime") + if objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // Note: a similar change also appears in + // cmd/dist/build.go. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go index 4c1f36dbd6..ade8964b7c 100644 --- a/src/cmd/go/internal/work/gccgo.go +++ b/src/cmd/go/internal/work/gccgo.go @@ -11,11 +11,14 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" + "cmd/internal/pkgpath" ) // The Gccgo toolchain. @@ -91,13 +94,37 @@ func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg args = append(args, "-I", root) } } - if cfg.BuildTrimpath && b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { - args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") - args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + + if b.gccSupportsFlag(args[:1], "-ffile-prefix-map=a=b") { + if cfg.BuildTrimpath { + args = append(args, "-ffile-prefix-map="+base.Cwd+"=.") + args = append(args, "-ffile-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if fsys.OverlayFile != "" { + for _, name := range gofiles { + absPath := mkAbs(p.Dir, name) + overlayPath, ok := fsys.OverlayPath(absPath) + if !ok { + continue + } + toPath := absPath + // gccgo only applies the last matching rule, so also handle the case where + // BuildTrimpath is true and the path is relative to base.Cwd. + if cfg.BuildTrimpath && str.HasFilePathPrefix(toPath, base.Cwd) { + toPath = "." + toPath[len(base.Cwd):] + } + args = append(args, "-ffile-prefix-map="+overlayPath+"="+toPath) + } + } } + args = append(args, a.Package.Internal.Gccgoflags...) for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) + f := mkAbs(p.Dir, f) + // Overlay files if necessary. + // See comment on gctoolchain.gc about overlay TODOs + f, _ = fsys.OverlayPath(f) + args = append(args, f) } output, err = b.runOut(a, p.Dir, nil, args) @@ -174,7 +201,7 @@ func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]strin ofiles = append(ofiles, ofile) sfile = mkAbs(p.Dir, sfile) defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) } defs = tools.maybePIC(defs) @@ -531,7 +558,7 @@ func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error cfile = mkAbs(p.Dir, cfile) defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} defs = append(defs, b.gccArchArgs()...) - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + if pkgpath := tools.gccgoCleanPkgpath(b, p); pkgpath != "" { defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) } compiler := envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) @@ -568,14 +595,19 @@ func gccgoPkgpath(p *load.Package) string { return p.ImportPath } -func gccgoCleanPkgpath(p *load.Package) string { - clean := func(r rune) rune { - switch { - case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', - '0' <= r && r <= '9': - return r +var gccgoToSymbolFuncOnce sync.Once +var gccgoToSymbolFunc func(string) string + +func (tools gccgoToolchain) gccgoCleanPkgpath(b *Builder, p *load.Package) string { + gccgoToSymbolFuncOnce.Do(func() { + fn, err := pkgpath.ToSymbolFunc(tools.compiler(), b.WorkDir) + if err != nil { + fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + base.SetExitStatus(2) + base.Exit() } - return '_' - } - return strings.Map(clean, gccgoPkgpath(p)) + gccgoToSymbolFunc = fn + }) + + return gccgoToSymbolFunc(gccgoPkgpath(p)) } diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index b0d6133768..d65c076c6a 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -9,6 +9,7 @@ package work import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/modload" "cmd/internal/objabi" "cmd/internal/sys" @@ -24,6 +25,9 @@ func BuildInit() { modload.Init() instrumentInit() buildModeInit() + if err := fsys.Init(base.Cwd); err != nil { + base.Fatalf("go: %v", err) + } // Make sure -pkgdir is absolute, because we run commands // in different directories. @@ -37,6 +41,13 @@ func BuildInit() { cfg.BuildPkgdir = p } + // Make sure CC and CXX are absolute paths + for _, key := range []string{"CC", "CXX"} { + if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { + base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) + } + } + // For each experiment that has been enabled in the toolchain, define a // build tag with the same name but prefixed by "goexperiment." which can be // used for compiling alternative files for the experiment. This allows @@ -157,7 +168,10 @@ func buildModeInit() { ldBuildmode = "pie" case "windows": ldBuildmode = "pie" - case "darwin", "ios": + case "ios": + codegenArg = "-shared" + ldBuildmode = "pie" + case "darwin": switch cfg.Goarch { case "arm64": codegenArg = "-shared" diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index d2a2697f0f..bcc29c8cbe 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -132,6 +132,7 @@ var validCompilerFlagsWithNextArg = []string{ "-U", "-I", "-framework", + "-include", "-isysroot", "-isystem", "--sysroot", diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go index 11e74f29c6..43a0ab1e47 100644 --- a/src/cmd/go/internal/work/security_test.go +++ b/src/cmd/go/internal/work/security_test.go @@ -62,6 +62,8 @@ var goodCompilerFlags = [][]string{ {"-I", "=/usr/include/libxml2"}, {"-I", "dir"}, {"-I", "$SYSROOT/dir"}, + {"-isystem", "/usr/include/mozjs-68"}, + {"-include", "/usr/include/mozjs-68/RequiredDefines.h"}, {"-framework", "Chocolate"}, {"-x", "c"}, {"-v"}, @@ -91,6 +93,7 @@ var badCompilerFlags = [][]string{ {"-I", "@foo"}, {"-I", "-foo"}, {"-I", "=@obj"}, + {"-include", "@foo"}, {"-framework", "-Caffeine"}, {"-framework", "@Home"}, {"-x", "--c"}, |