From ab2a5b48665eed6d670d719cdef5335bc3602359 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 11 Aug 2020 12:57:01 -0400 Subject: cmd/go: add basic support for overlays This CL adds basic support for listing packages with overlays. The new cmd/go/internal/fs package adds an abstraction for communicating with the file system that will open files according to their overlaid paths, and provides functions to override those in the build context to open overlaid files. There is also some support for executing builds on packages with overlays. In cmd/go/internal/work.(*Builder).build, paths are mapped to their overlaid paths before they are given as arguments to tools. For #39958 Change-Id: I5ec0eb9ebbca303e2f1e7dbe22ec32613bc1fd17 Reviewed-on: https://go-review.googlesource.com/c/go/+/253747 Trust: Michael Matloob Trust: Jay Conrod Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/fsys/fsys.go | 426 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 src/cmd/go/internal/fsys/fsys.go (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go new file mode 100644 index 0000000000..d64ce0aba1 --- /dev/null +++ b/src/cmd/go/internal/fsys/fsys.go @@ -0,0 +1,426 @@ +// Package fsys is an abstraction for reading files that +// allows for virtual overlays on top of the files on disk. +package fsys + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// OverlayFile is the path to a text file in the OverlayJSON format. +// It is the value of the -overlay flag. +var OverlayFile string + +// OverlayJSON is the format overlay files are expected to be in. +// The Replace map maps from overlaid paths to replacement paths: +// the Go command will forward all reads trying to open +// each overlaid path to its replacement path, or consider the overlaid +// path not to exist if the replacement path is empty. +type OverlayJSON struct { + Replace map[string]string +} + +type node struct { + actualFilePath string // empty if a directory + children map[string]*node // path element → file or directory +} + +func (n *node) isDir() bool { + return n.actualFilePath == "" && n.children != nil +} + +func (n *node) isDeleted() bool { + return n.actualFilePath == "" && n.children == nil +} + +// TODO(matloob): encapsulate these in an io/fs-like interface +var overlay map[string]*node // path -> file or directory node +var cwd string // copy of base.Cwd to avoid dependency + +// Canonicalize a path for looking it up in the overlay. +// Important: filepath.Join(cwd, path) doesn't always produce +// the correct absolute path if path is relative, because on +// Windows producing the correct absolute path requires making +// a syscall. So this should only be used when looking up paths +// in the overlay, or canonicalizing the paths in the overlay. +func canonicalize(path string) string { + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { + // On Windows filepath.Join(cwd, path) doesn't always work. In general + // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go + // use filepath.Join(cwd, path), but cmd/go specifically supports Windows + // paths that start with "\" which implies the path is relative to the + // volume of the working directory. See golang.org/issue/8130. + return filepath.Join(v, path) + } + + // Make the path absolute. + return filepath.Join(cwd, path) +} + +// Init initializes the overlay, if one is being used. +func Init(wd string) error { + if overlay != nil { + // already initialized + return nil + } + + cwd = wd + + if OverlayFile == "" { + return nil + } + + b, err := ioutil.ReadFile(OverlayFile) + if err != nil { + return fmt.Errorf("reading overlay file: %v", err) + } + + var overlayJSON OverlayJSON + if err := json.Unmarshal(b, &overlayJSON); err != nil { + return fmt.Errorf("parsing overlay JSON: %v", err) + } + + return initFromJSON(overlayJSON) +} + +func initFromJSON(overlayJSON OverlayJSON) error { + // Canonicalize the paths in in the overlay map. + // Use reverseCanonicalized to check for collisions: + // no two 'from' paths should canonicalize to the same path. + overlay = make(map[string]*node) + reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates + // Build a table of file and directory nodes from the replacement map. + + // Remove any potential non-determinism from iterating over map by sorting it. + replaceFrom := make([]string, 0, len(overlayJSON.Replace)) + for k := range overlayJSON.Replace { + replaceFrom = append(replaceFrom, k) + } + sort.Strings(replaceFrom) + + for _, from := range replaceFrom { + to := overlayJSON.Replace[from] + // Canonicalize paths and check for a collision. + if from == "" { + return fmt.Errorf("empty string key in overlay file Replace map") + } + cfrom := canonicalize(from) + if to != "" { + // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". + to = canonicalize(to) + } + if otherFrom, seen := reverseCanonicalized[cfrom]; seen { + return fmt.Errorf( + "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) + } + reverseCanonicalized[cfrom] = from + from = cfrom + + // Create node for overlaid file. + dir, base := filepath.Dir(from), filepath.Base(from) + if n, ok := overlay[from]; ok { + // All 'from' paths in the overlay are file paths. Since the from paths + // are in a map, they are unique, so if the node already exists we added + // it below when we create parent directory nodes. That is, that + // both a file and a path to one of its parent directories exist as keys + // in the Replace map. + // + // This only applies if the overlay directory has any files or directories + // in it: placeholder directories that only contain deleted files don't + // count. They are safe to be overwritten with actual files. + for _, f := range n.children { + if !f.isDeleted() { + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) + } + } + } + overlay[from] = &node{actualFilePath: to} + + // Add parent directory nodes to overlay structure. + childNode := overlay[from] + for { + dirNode := overlay[dir] + if dirNode == nil || dirNode.isDeleted() { + dirNode = &node{children: make(map[string]*node)} + overlay[dir] = dirNode + } + if childNode.isDeleted() { + // Only create one parent for a deleted file: + // the directory only conditionally exists if + // there are any non-deleted children, so + // we don't create their parents. + if dirNode.isDir() { + dirNode.children[base] = childNode + } + break + } + if !dirNode.isDir() { + // This path already exists as a file, so it can't be a parent + // directory. See comment at error above. + return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) + } + dirNode.children[base] = childNode + parent := filepath.Dir(dir) + if parent == dir { + break // reached the top; there is no parent + } + dir, base = parent, filepath.Base(dir) + childNode = dirNode + } + } + + return nil +} + +// IsDir returns true if path is a directory on disk or in the +// overlay. +func IsDir(path string) (bool, error) { + path = canonicalize(path) + + if _, ok := parentIsOverlayFile(path); ok { + return false, nil + } + + if n, ok := overlay[path]; ok { + return n.isDir(), nil + } + + fi, err := os.Stat(path) + if err != nil { + return false, err + } + + return fi.IsDir(), nil +} + +// parentIsOverlayFile returns whether name or any of +// its parents are directories in the overlay, and the first parent found, +// including name itself, that's a directory in the overlay. +func parentIsOverlayFile(name string) (string, bool) { + if overlay != nil { + // Check if name can't possibly be a directory because + // it or one of its parents is overlaid with a file. + // TODO(matloob): Maybe save this to avoid doing it every time? + prefix := name + for { + node := overlay[prefix] + if node != nil && !node.isDir() { + return prefix, true + } + parent := filepath.Dir(prefix) + if parent == prefix { + break + } + prefix = parent + } + } + + return "", false +} + +// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles +// that the argument is not a directory, so that IsDirWithGoFiles doesn't +// return an error. +var errNotDir = errors.New("not a directory") + +// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. +// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory +// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). +func readDir(dir string) ([]os.FileInfo, error) { + fis, err := ioutil.ReadDir(dir) + if err == nil { + return fis, nil + } + + if os.IsNotExist(err) { + return nil, err + } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } else { + return nil, err + } +} + +// ReadDir provides a slice of os.FileInfo entries corresponding +// to the overlaid files in the directory. +func ReadDir(dir string) ([]os.FileInfo, error) { + dir = canonicalize(dir) + if _, ok := parentIsOverlayFile(dir); ok { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + } + + dirNode := overlay[dir] + if dirNode == nil { + return readDir(dir) + } else if dirNode.isDeleted() { + return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: os.ErrNotExist} + } + diskfis, err := readDir(dir) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { + return nil, err + } + + // Stat files in overlay to make composite list of fileinfos + files := make(map[string]os.FileInfo) + for _, f := range diskfis { + files[f.Name()] = f + } + for name, to := range dirNode.children { + switch { + case to.isDir(): + files[name] = fakeDir(name) + case to.isDeleted(): + delete(files, name) + default: + // This is a regular file. + f, err := os.Lstat(to.actualFilePath) + if err != nil { + files[name] = missingFile(name) + continue + } else if f.IsDir() { + return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", + filepath.Join(dir, name), to.actualFilePath) + } + // Add a fileinfo for the overlaid file, so that it has + // the original file's name, but the overlaid file's metadata. + files[name] = fakeFile{name, f} + } + } + sortedFiles := diskfis[:0] + for _, f := range files { + sortedFiles = append(sortedFiles, f) + } + sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) + return sortedFiles, nil +} + +// OverlayPath returns the path to the overlaid contents of the +// file, the empty string if the overlay deletes the file, or path +// itself if the file is not in the overlay, the file is a directory +// in the overlay, or there is no overlay. +// It returns true if the path is overlaid with a regular file +// or deleted, and false otherwise. +func OverlayPath(path string) (string, bool) { + if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { + return p.actualFilePath, ok + } + + return path, false +} + +// Open opens the file at or overlaid on the given path. +func Open(path string) (*os.File, error) { + cpath := canonicalize(path) + if node, ok := overlay[cpath]; ok { + if node.isDir() { + return nil, &os.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} + } + return os.Open(node.actualFilePath) + } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + // The file is deleted explicitly in the Replace map, + // or implicitly because one of its parent directories was + // replaced by a file. + return nil, &os.PathError{ + Op: "Open", + Path: path, + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} + } else { + return os.Open(cpath) + } +} + +// IsDirWithGoFiles reports whether dir is a directory containing Go files +// either on disk or in the overlay. +func IsDirWithGoFiles(dir string) (bool, error) { + fis, err := ReadDir(dir) + if os.IsNotExist(err) || errors.Is(err, errNotDir) { + return false, nil + } else if err != nil { + return false, err + } + + var firstErr error + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // TODO(matloob): this enforces that the "from" in the map + // has a .go suffix, but the actual destination file + // doesn't need to have a .go suffix. Is this okay with the + // compiler? + if !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if fi.Mode().IsRegular() { + return true, nil + } + + // fi is the result of an Lstat, so it doesn't follow symlinks. + // But it's okay if the file is a symlink pointing to a regular + // file, so use os.Stat to follow symlinks and check that. + actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) + if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { + return true, nil + } else if err != nil && firstErr == nil { + firstErr = err + } + } + + // No go files found in directory. + return false, firstErr +} + +// fakeFile provides an os.FileInfo implementation for an overlaid file, +// so that the file has the name of the overlaid file, but takes all +// other characteristics of the replacement file. +type fakeFile struct { + name string + real os.FileInfo +} + +func (f fakeFile) Name() string { return f.name } +func (f fakeFile) Size() int64 { return f.real.Size() } +func (f fakeFile) Mode() os.FileMode { return f.real.Mode() } +func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } +func (f fakeFile) IsDir() bool { return f.real.IsDir() } +func (f fakeFile) Sys() interface{} { return f.real.Sys() } + +// missingFile provides an os.FileInfo for an overlaid file where the +// destination file in the overlay doesn't exist. It returns zero values +// for the fileInfo methods other than Name, set to the file's name, and Mode +// set to ModeIrregular. +type missingFile string + +func (f missingFile) Name() string { return string(f) } +func (f missingFile) Size() int64 { return 0 } +func (f missingFile) Mode() os.FileMode { return os.ModeIrregular } +func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } +func (f missingFile) IsDir() bool { return false } +func (f missingFile) Sys() interface{} { return nil } + +// fakeDir provides an os.FileInfo implementation for directories that are +// implicitly created by overlaid files. Each directory in the +// path of an overlaid file is considered to exist in the overlay filesystem. +type fakeDir string + +func (f fakeDir) Name() string { return string(f) } +func (f fakeDir) Size() int64 { return 0 } +func (f fakeDir) Mode() os.FileMode { return os.ModeDir | 0500 } +func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } +func (f fakeDir) IsDir() bool { return true } +func (f fakeDir) Sys() interface{} { return nil } -- cgit v1.2.1 From 15a11cedc6a9aac722369f134b76a157a559e050 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 6 Oct 2020 13:16:46 -0400 Subject: cmd/go: support walking through overlay directories Change-Id: I7d9d75aa1dbc34fec5073ca36091c626b9dd4920 Reviewed-on: https://go-review.googlesource.com/c/go/+/261537 Trust: Michael Matloob Trust: Jay Conrod Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/fsys/fsys.go | 78 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index d64ce0aba1..489af93496 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -208,8 +208,8 @@ func IsDir(path string) (bool, error) { } // parentIsOverlayFile returns whether name or any of -// its parents are directories in the overlay, and the first parent found, -// including name itself, that's a directory in the overlay. +// its parents are files in the overlay, and the first parent found, +// including name itself, that's a file in the overlay. func parentIsOverlayFile(name string) (string, bool) { if overlay != nil { // Check if name can't possibly be a directory because @@ -385,6 +385,80 @@ func IsDirWithGoFiles(dir string) (bool, error) { return false, firstErr } +// walk recursively descends path, calling walkFn. Copied, with some +// modifications from path/filepath.walk. +func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + fis, readErr := ReadDir(path) + walkErr := walkFn(path, info, readErr) + // If readErr != nil, walk can't walk into this directory. + // walkErr != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of readErr and walkErr isn't nil, walk will return. + if readErr != nil || walkErr != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore readErr and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return walkErr + } + + for _, fi := range fis { + filename := filepath.Join(path, fi.Name()) + if walkErr = walk(filename, fi, walkFn); walkErr != nil { + if !fi.IsDir() || walkErr != filepath.SkipDir { + return walkErr + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + info, err := lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(root, info, walkFn) + } + if err == filepath.SkipDir { + return nil + } + return err +} + +// lstat implements a version of os.Lstat that operates on the overlay filesystem. +func lstat(path string) (os.FileInfo, error) { + cpath := canonicalize(path) + + if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + } + + node, ok := overlay[cpath] + if !ok { + // The file or directory is not overlaid. + return os.Lstat(cpath) + } + + switch { + case node.isDeleted(): + return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + case node.isDir(): + return fakeDir(filepath.Base(cpath)), nil + default: + fi, err := os.Lstat(node.actualFilePath) + if err != nil { + return nil, err + } + return fakeFile{name: filepath.Base(cpath), real: fi}, nil + } +} + // fakeFile provides an os.FileInfo implementation for an overlaid file, // so that the file has the name of the overlaid file, but takes all // other characteristics of the replacement file. -- cgit v1.2.1 From 7bb721b9384bdd196befeaed593b185f7f2a5589 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 7 Jul 2020 13:49:21 -0400 Subject: all: update references to symbols moved from os to io/fs The old os references are still valid, but update our code to reflect best practices and get used to the new locations. Code compiled with the bootstrap toolchain (cmd/asm, cmd/dist, cmd/compile, debug/elf) must remain Go 1.4-compatible and is excluded. For #41190. Change-Id: I8f9526977867c10a221e2f392f78d7dec073f1bd Reviewed-on: https://go-review.googlesource.com/c/go/+/243907 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Rob Pike --- src/cmd/go/internal/fsys/fsys.go | 41 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 489af93496..67359ffb6d 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -240,7 +241,7 @@ var errNotDir = errors.New("not a directory") // readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. // Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory // can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). -func readDir(dir string) ([]os.FileInfo, error) { +func readDir(dir string) ([]fs.FileInfo, error) { fis, err := ioutil.ReadDir(dir) if err == nil { return fis, nil @@ -249,25 +250,25 @@ func readDir(dir string) ([]os.FileInfo, error) { if os.IsNotExist(err) { return nil, err } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} } else { return nil, err } } -// ReadDir provides a slice of os.FileInfo entries corresponding +// ReadDir provides a slice of fs.FileInfo entries corresponding // to the overlaid files in the directory. -func ReadDir(dir string) ([]os.FileInfo, error) { +func ReadDir(dir string) ([]fs.FileInfo, error) { dir = canonicalize(dir) if _, ok := parentIsOverlayFile(dir); ok { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} } dirNode := overlay[dir] if dirNode == nil { return readDir(dir) } else if dirNode.isDeleted() { - return nil, &os.PathError{Op: "ReadDir", Path: dir, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} } diskfis, err := readDir(dir) if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { @@ -275,7 +276,7 @@ func ReadDir(dir string) ([]os.FileInfo, error) { } // Stat files in overlay to make composite list of fileinfos - files := make(map[string]os.FileInfo) + files := make(map[string]fs.FileInfo) for _, f := range diskfis { files[f.Name()] = f } @@ -327,14 +328,14 @@ func Open(path string) (*os.File, error) { cpath := canonicalize(path) if node, ok := overlay[cpath]; ok { if node.isDir() { - return nil, &os.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} + return nil, &fs.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} } return os.Open(node.actualFilePath) } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { // The file is deleted explicitly in the Replace map, // or implicitly because one of its parent directories was // replaced by a file. - return nil, &os.PathError{ + return nil, &fs.PathError{ Op: "Open", Path: path, Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} @@ -387,7 +388,7 @@ func IsDirWithGoFiles(dir string) (bool, error) { // walk recursively descends path, calling walkFn. Copied, with some // modifications from path/filepath.walk. -func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { +func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { if !info.IsDir() { return walkFn(path, info, nil) } @@ -432,11 +433,11 @@ func Walk(root string, walkFn filepath.WalkFunc) error { } // lstat implements a version of os.Lstat that operates on the overlay filesystem. -func lstat(path string) (os.FileInfo, error) { +func lstat(path string) (fs.FileInfo, error) { cpath := canonicalize(path) if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { - return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} } node, ok := overlay[cpath] @@ -447,7 +448,7 @@ func lstat(path string) (os.FileInfo, error) { switch { case node.isDeleted(): - return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist} + return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} case node.isDir(): return fakeDir(filepath.Base(cpath)), nil default: @@ -459,22 +460,22 @@ func lstat(path string) (os.FileInfo, error) { } } -// fakeFile provides an os.FileInfo implementation for an overlaid file, +// fakeFile provides an fs.FileInfo implementation for an overlaid file, // so that the file has the name of the overlaid file, but takes all // other characteristics of the replacement file. type fakeFile struct { name string - real os.FileInfo + real fs.FileInfo } func (f fakeFile) Name() string { return f.name } func (f fakeFile) Size() int64 { return f.real.Size() } -func (f fakeFile) Mode() os.FileMode { return f.real.Mode() } +func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } func (f fakeFile) IsDir() bool { return f.real.IsDir() } func (f fakeFile) Sys() interface{} { return f.real.Sys() } -// missingFile provides an os.FileInfo for an overlaid file where the +// missingFile provides an fs.FileInfo for an overlaid file where the // destination file in the overlay doesn't exist. It returns zero values // for the fileInfo methods other than Name, set to the file's name, and Mode // set to ModeIrregular. @@ -482,19 +483,19 @@ type missingFile string func (f missingFile) Name() string { return string(f) } func (f missingFile) Size() int64 { return 0 } -func (f missingFile) Mode() os.FileMode { return os.ModeIrregular } +func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } func (f missingFile) IsDir() bool { return false } func (f missingFile) Sys() interface{} { return nil } -// fakeDir provides an os.FileInfo implementation for directories that are +// fakeDir provides an fs.FileInfo implementation for directories that are // implicitly created by overlaid files. Each directory in the // path of an overlaid file is considered to exist in the overlay filesystem. type fakeDir string func (f fakeDir) Name() string { return string(f) } func (f fakeDir) Size() int64 { return 0 } -func (f fakeDir) Mode() os.FileMode { return os.ModeDir | 0500 } +func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } func (f fakeDir) IsDir() bool { return true } func (f fakeDir) Sys() interface{} { return nil } -- cgit v1.2.1 From 5e9582e3f0d10523d32a25a338cbade21266dca3 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 15 Oct 2020 11:45:32 -0400 Subject: cmd/go: support overlays for synthesized packages. The main missing piece here was supporting Stat in the overlay filesystem, in the parts of the package code that determines whether an command line argument is a file on disk or a directory. so this change adds a Stat function to the fsys package. It's implemented the same way as the already existing fsys.lstat function, but instead of os.Lstat, it calls os.Stat on disk files. Then, the change changes parts of the package code to use the overlay Stat instead of the os package's Stat. For #39958 Change-Id: I8e478ae386f05b48d7dd71bd7e47584f090623df Reviewed-on: https://go-review.googlesource.com/c/go/+/262617 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/fsys/fsys.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 67359ffb6d..814e323701 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -434,29 +434,39 @@ func Walk(root string, walkFn filepath.WalkFunc) error { // lstat implements a version of os.Lstat that operates on the overlay filesystem. func lstat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Lstat, "lstat") +} + +// Stat implements a version of os.Stat that operates on the overlay filesystem. +func Stat(path string) (fs.FileInfo, error) { + return overlayStat(path, os.Stat, "stat") +} + +// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). +func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { cpath := canonicalize(path) if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { - return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} + return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} } node, ok := overlay[cpath] if !ok { // The file or directory is not overlaid. - return os.Lstat(cpath) + return osStat(path) } switch { case node.isDeleted(): return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} case node.isDir(): - return fakeDir(filepath.Base(cpath)), nil + return fakeDir(filepath.Base(path)), nil default: - fi, err := os.Lstat(node.actualFilePath) + fi, err := osStat(node.actualFilePath) if err != nil { return nil, err } - return fakeFile{name: filepath.Base(cpath), real: fi}, nil + return fakeFile{name: filepath.Base(path), real: fi}, nil } } -- cgit v1.2.1 From a8e2966eb01f175c330f6669f838e83af2cb73e3 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 22 Oct 2020 20:55:38 -0400 Subject: cmd/go/internal/fsys: rewrite non-idiomatic if statements https://golang.org/doc/effective_go.html#if Change-Id: I4d868e05c7827638f45b3b06d8762f5a298d56f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/264537 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/internal/fsys/fsys.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 814e323701..3275c3faf7 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -249,11 +249,11 @@ func readDir(dir string) ([]fs.FileInfo, error) { if os.IsNotExist(err) { return nil, err - } else if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { + } + if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} - } else { - return nil, err } + return nil, err } // ReadDir provides a slice of fs.FileInfo entries corresponding @@ -267,7 +267,8 @@ func ReadDir(dir string) ([]fs.FileInfo, error) { dirNode := overlay[dir] if dirNode == nil { return readDir(dir) - } else if dirNode.isDeleted() { + } + if dirNode.isDeleted() { return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} } diskfis, err := readDir(dir) @@ -331,17 +332,18 @@ func Open(path string) (*os.File, error) { return nil, &fs.PathError{Op: "Open", Path: path, Err: errors.New("fsys.Open doesn't support opening directories yet")} } return os.Open(node.actualFilePath) - } else if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { + } + if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { // The file is deleted explicitly in the Replace map, // or implicitly because one of its parent directories was // replaced by a file. return nil, &fs.PathError{ Op: "Open", Path: path, - Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent)} - } else { - return os.Open(cpath) + Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), + } } + return os.Open(cpath) } // IsDirWithGoFiles reports whether dir is a directory containing Go files @@ -350,7 +352,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { fis, err := ReadDir(dir) if os.IsNotExist(err) || errors.Is(err, errNotDir) { return false, nil - } else if err != nil { + } + if err != nil { return false, err } @@ -377,7 +380,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { return true, nil - } else if err != nil && firstErr == nil { + } + if err != nil && firstErr == nil { firstErr = err } } -- cgit v1.2.1 From 69496a22682108bed606d4d509cfa3253f0cac3b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 27 Oct 2020 10:41:25 -0400 Subject: cmd/go: fix bug introduced in CL 264537 Shadowing bug noted after submit by Tom Thorogood. Change-Id: I5f40cc3863dcd7dba5469f8530e9d0460e7c3e7e Reviewed-on: https://go-review.googlesource.com/c/go/+/265537 Trust: Russ Cox Run-TryBot: Russ Cox Reviewed-by: Jay Conrod TryBot-Result: Go Bot --- src/cmd/go/internal/fsys/fsys.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 3275c3faf7..5a8b36e2bc 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -378,7 +378,8 @@ func IsDirWithGoFiles(dir string) (bool, error) { // But it's okay if the file is a symlink pointing to a regular // file, so use os.Stat to follow symlinks and check that. actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) - if fi, err := os.Stat(actualFilePath); err == nil && fi.Mode().IsRegular() { + fi, err := os.Stat(actualFilePath) + if err == nil && fi.Mode().IsRegular() { return true, nil } if err != nil && firstErr == nil { -- cgit v1.2.1 From 3c55aea67aa65c62016020d5907b481da010f7e0 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 22 Oct 2020 21:33:02 -0400 Subject: cmd/go/internal/fsys: add Glob Glob is needed for //go:embed processing. Also change TestReadDir to be deterministic and print more output about failures. Change-Id: Ie22a9c5b32bda753579ff98cec1d28e3244c4e06 Reviewed-on: https://go-review.googlesource.com/c/go/+/264538 Trust: Russ Cox Trust: Jay Conrod Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Michael Matloob --- src/cmd/go/internal/fsys/fsys.go | 163 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) (limited to 'src/cmd/go/internal/fsys/fsys.go') diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go index 5a8b36e2bc..44d9b1368b 100644 --- a/src/cmd/go/internal/fsys/fsys.go +++ b/src/cmd/go/internal/fsys/fsys.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "sort" "strings" "time" @@ -514,3 +515,165 @@ func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } func (f fakeDir) IsDir() bool { return true } func (f fakeDir) Sys() interface{} { return nil } + +// Glob is like filepath.Glob but uses the overlay file system. +func Glob(pattern string) (matches []string, err error) { + // Check pattern is well-formed. + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, err + } + if !hasMeta(pattern) { + if _, err = lstat(pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + volumeLen := 0 + if runtime.GOOS == "windows" { + volumeLen, dir = cleanGlobPathWindows(dir) + } else { + dir = cleanGlobPath(dir) + } + + if !hasMeta(dir[volumeLen:]) { + return glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, filepath.ErrBadPattern + } + + var m []string + m, err = Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(filepath.Separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +func volumeNameLen(path string) int { + isSlash := func(c uint8) bool { + return c == '\\' || c == '/' + } + if len(path) < 2 { + return 0 + } + // with drive letter + c := path[0] + if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return 2 + } + // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && + !isSlash(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if isSlash(path[n]) { + n++ + // third, following something characters. its share name. + if !isSlash(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if isSlash(path[n]) { + break + } + } + return n + } + break + } + } + } + return 0 +} + +// cleanGlobPathWindows is windows version of cleanGlobPath. +func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { + vollen := volumeNameLen(path) + switch { + case path == "": + return 0, "." + case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ + // do nothing to the path + return vollen + 1, path + case vollen == len(path) && len(path) == 2: // C: + return vollen, path + "." // convert C: into C:. + default: + if vollen >= len(path) { + vollen = len(path) - 1 + } + return vollen, path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := Stat(dir) + if err != nil { + return // ignore I/O error + } + if !fi.IsDir() { + return // ignore I/O error + } + + list, err := ReadDir(dir) + if err != nil { + return // ignore I/O error + } + + var names []string + for _, info := range list { + names = append(names, info.Name()) + } + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by filepath.Match. +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} -- cgit v1.2.1