diff options
author | Ian Lance Taylor <iant@golang.org> | 2022-02-11 14:53:56 -0800 |
---|---|---|
committer | Ian Lance Taylor <iant@golang.org> | 2022-02-11 15:01:19 -0800 |
commit | 8dc2499aa62f768c6395c9754b8cabc1ce25c494 (patch) | |
tree | 43d7fd2bbfd7ad8c9625a718a5e8718889351994 /libgo/go/cmd/go/internal | |
parent | 9a56779dbc4e2d9c15be8d31e36f2f59be7331a8 (diff) | |
download | gcc-8dc2499aa62f768c6395c9754b8cabc1ce25c494.tar.gz |
libgo: update to Go1.18beta2
gotools/
* Makefile.am (go_cmd_cgo_files): Add ast_go118.go
(check-go-tool): Copy golang.org/x/tools directories.
* Makefile.in: Regenerate.
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/384695
Diffstat (limited to 'libgo/go/cmd/go/internal')
124 files changed, 4835 insertions, 2699 deletions
diff --git a/libgo/go/cmd/go/internal/base/base.go b/libgo/go/cmd/go/internal/base/base.go index 954ce47a989..c2d4e6b2588 100644 --- a/libgo/go/cmd/go/internal/base/base.go +++ b/libgo/go/cmd/go/internal/base/base.go @@ -117,12 +117,12 @@ func Exit() { os.Exit(exitStatus) } -func Fatalf(format string, args ...interface{}) { +func Fatalf(format string, args ...any) { Errorf(format, args...) Exit() } -func Errorf(format string, args ...interface{}) { +func Errorf(format string, args ...any) { log.Printf(format, args...) SetExitStatus(1) } @@ -151,7 +151,7 @@ func GetExitStatus() int { // Run runs the command, with stdout and stderr // connected to the go command's own stdout and stderr. // If the command fails, Run reports the error using Errorf. -func Run(cmdargs ...interface{}) { +func Run(cmdargs ...any) { cmdline := str.StringList(cmdargs...) if cfg.BuildN || cfg.BuildX { fmt.Printf("%s\n", strings.Join(cmdline, " ")) diff --git a/libgo/go/cmd/go/internal/base/flag.go b/libgo/go/cmd/go/internal/base/flag.go index 677f8196827..2c72c7e562b 100644 --- a/libgo/go/cmd/go/internal/base/flag.go +++ b/libgo/go/cmd/go/internal/base/flag.go @@ -9,7 +9,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" - "cmd/go/internal/str" + "cmd/internal/quoted" ) // A StringsFlag is a command-line flag that interprets its argument @@ -18,7 +18,7 @@ type StringsFlag []string func (v *StringsFlag) Set(s string) error { var err error - *v, err = str.SplitQuotedFields(s) + *v, err = quoted.Split(s) if *v == nil { *v = []string{} } @@ -62,6 +62,13 @@ func AddModFlag(flags *flag.FlagSet) { flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") } +// AddWorkfileFlag adds the workfile flag to the flag set. It enables workspace +// mode for commands that support it by resetting the cfg.WorkFile variable +// to "" (equivalent to auto) rather than off. +func AddWorkfileFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.WorkFile, explicit: &cfg.WorkFileExplicit}, "workfile", "") +} + // AddModCommonFlags adds the module-related flags common to build commands // and 'go mod' subcommands. func AddModCommonFlags(flags *flag.FlagSet) { diff --git a/libgo/go/cmd/go/internal/base/signal_notunix.go b/libgo/go/cmd/go/internal/base/signal_notunix.go index 5cc0b0f1011..682705f9b2c 100644 --- a/libgo/go/cmd/go/internal/base/signal_notunix.go +++ b/libgo/go/cmd/go/internal/base/signal_notunix.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build plan9 || windows -// +build plan9 windows package base diff --git a/libgo/go/cmd/go/internal/base/signal_unix.go b/libgo/go/cmd/go/internal/base/signal_unix.go index cdb25934c2d..0cf58eb9299 100644 --- a/libgo/go/cmd/go/internal/base/signal_unix.go +++ b/libgo/go/cmd/go/internal/base/signal_unix.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build aix || darwin || dragonfly || freebsd || hurd || js || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd hurd js linux netbsd openbsd solaris package base diff --git a/libgo/go/cmd/go/internal/base/tool.go b/libgo/go/cmd/go/internal/base/tool.go index d0da65e03ce..f9270169650 100644 --- a/libgo/go/cmd/go/internal/base/tool.go +++ b/libgo/go/cmd/go/internal/base/tool.go @@ -36,7 +36,7 @@ func Tool(toolName string) string { } // Give a nice message if there is no tool with that name. if _, err := os.Stat(toolPath); err != nil { - fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName) + fmt.Fprintf(os.Stderr, "go: no such tool %q\n", toolName) SetExitStatus(2) Exit() } diff --git a/libgo/go/cmd/go/internal/bug/bug.go b/libgo/go/cmd/go/internal/bug/bug.go index 307527c695c..702dc2a14ac 100644 --- a/libgo/go/cmd/go/internal/bug/bug.go +++ b/libgo/go/cmd/go/internal/bug/bug.go @@ -40,7 +40,7 @@ func init() { func runBug(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { - base.Fatalf("go bug: bug takes no arguments") + base.Fatalf("go: bug takes no arguments") } var buf bytes.Buffer buf.WriteString(bugHeader) @@ -106,8 +106,9 @@ func printGoEnv(w io.Writer) { } func printGoDetails(w io.Writer) { - printCmdOut(w, "GOROOT/bin/go version: ", filepath.Join(runtime.GOROOT(), "bin/go"), "version") - printCmdOut(w, "GOROOT/bin/go tool compile -V: ", filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "compile", "-V") + gocmd := filepath.Join(runtime.GOROOT(), "bin/go") + printCmdOut(w, "GOROOT/bin/go version: ", gocmd, "version") + printCmdOut(w, "GOROOT/bin/go tool compile -V: ", gocmd, "tool", "compile", "-V") } func printOSDetails(w io.Writer) { diff --git a/libgo/go/cmd/go/internal/cache/cache.go b/libgo/go/cmd/go/internal/cache/cache.go index d592d704978..93d7c25658f 100644 --- a/libgo/go/cmd/go/internal/cache/cache.go +++ b/libgo/go/cmd/go/internal/cache/cache.go @@ -533,3 +533,15 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { return nil } + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +// +// TODO(#48526): make Trim remove unused files from this directory. +func (c *Cache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/libgo/go/cmd/go/internal/cache/default.go b/libgo/go/cmd/go/internal/cache/default.go index 0b1c1e0c203..426dddfb978 100644 --- a/libgo/go/cmd/go/internal/cache/default.go +++ b/libgo/go/cmd/go/internal/cache/default.go @@ -30,6 +30,7 @@ var ( // README as a courtesy to explain where it came from. const cacheREADME = `This directory holds cached build artifacts from the Go build system. Run "go clean -cache" if the directory is getting too large. +Run "go clean -fuzzcache" to delete the fuzz cache. See golang.org to learn more about Go. ` diff --git a/libgo/go/cmd/go/internal/cfg/cfg.go b/libgo/go/cmd/go/internal/cfg/cfg.go index 57a3c1ff6fb..7f68d7bb628 100644 --- a/libgo/go/cmd/go/internal/cfg/cfg.go +++ b/libgo/go/cmd/go/internal/cfg/cfg.go @@ -26,6 +26,7 @@ import ( var ( BuildA bool // -a flag BuildBuildmode string // -buildmode flag + BuildBuildvcs bool // -buildvcs flag BuildContext = defaultContext() BuildMod string // -mod flag BuildModExplicit bool // whether -mod was set explicitly @@ -33,6 +34,7 @@ var ( BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag + BuildASan bool // -asan flag BuildN bool // -n flag BuildO string // -o flag BuildP = runtime.GOMAXPROCS(0) // -p flag @@ -47,17 +49,26 @@ var ( BuildWork bool // -work flag BuildX bool // -x flag - ModCacheRW bool // -modcacherw flag - ModFile string // -modfile flag + ModCacheRW bool // -modcacherw flag + ModFile string // -modfile flag + WorkFile string // -workfile flag + WorkFileExplicit bool // whether -workfile was set explicitly CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) DebugTrace string // -debug-trace flag + + // GoPathError is set when GOPATH is not set. it contains an + // explanation why GOPATH is unset. + GoPathError string + + GOEXPERIMENT = envOr("GOEXPERIMENT", buildcfg.DefaultGOEXPERIMENT) ) func defaultContext() build.Context { ctxt := build.Default + ctxt.JoinPath = filepath.Join // back door to say "do not use go command" ctxt.GOROOT = findGOROOT() @@ -70,7 +81,7 @@ func defaultContext() build.Context { build.ToolDir = filepath.Join(ctxt.GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) } - ctxt.GOPATH = envOr("GOPATH", ctxt.GOPATH) + ctxt.GOPATH = envOr("GOPATH", gopath(ctxt)) // Override defaults computed in go/build with defaults // from go environment configuration file, if known. @@ -79,7 +90,7 @@ func defaultContext() build.Context { // The experiments flags are based on GOARCH, so they may // need to change. TODO: This should be cleaned up. - buildcfg.UpdateExperiments(ctxt.GOOS, ctxt.GOARCH, envOr("GOEXPERIMENT", buildcfg.DefaultGOEXPERIMENT)) + buildcfg.UpdateExperiments(ctxt.GOOS, ctxt.GOARCH, GOEXPERIMENT) ctxt.ToolTags = nil for _, exp := range buildcfg.EnabledExperiments() { ctxt.ToolTags = append(ctxt.ToolTags, "goexperiment."+exp) @@ -261,6 +272,7 @@ var ( // Used in envcmd.MkEnv and build ID computations. GOARM = envOr("GOARM", fmt.Sprint(buildcfg.GOARM)) GO386 = envOr("GO386", buildcfg.GO386) + GOAMD64 = envOr("GOAMD64", fmt.Sprintf("%s%d", "v", buildcfg.GOAMD64)) GOMIPS = envOr("GOMIPS", buildcfg.GOMIPS) GOMIPS64 = envOr("GOMIPS64", buildcfg.GOMIPS64) GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", buildcfg.GOPPC64)) @@ -287,6 +299,8 @@ func GetArchEnv() (key, val string) { return "GOARM", GOARM case "386": return "GO386", GO386 + case "amd64": + return "GOAMD64", GOAMD64 case "mips", "mipsle": return "GOMIPS", GOMIPS case "mips64", "mips64le": @@ -396,3 +410,24 @@ func gopathDir(rel string) string { } return filepath.Join(list[0], rel) } + +func gopath(ctxt build.Context) string { + if len(ctxt.GOPATH) > 0 { + return ctxt.GOPATH + } + env := "HOME" + if runtime.GOOS == "windows" { + env = "USERPROFILE" + } else if runtime.GOOS == "plan9" { + env = "home" + } + if home := os.Getenv(env); home != "" { + def := filepath.Join(home, "go") + if filepath.Clean(def) == filepath.Clean(runtime.GOROOT()) { + GoPathError = "cannot set GOROOT as GOPATH" + } + return "" + } + GoPathError = fmt.Sprintf("%s is not set", env) + return "" +} diff --git a/libgo/go/cmd/go/internal/clean/clean.go b/libgo/go/cmd/go/internal/clean/clean.go index fd4cb205591..dc93cdf5983 100644 --- a/libgo/go/cmd/go/internal/clean/clean.go +++ b/libgo/go/cmd/go/internal/clean/clean.go @@ -75,6 +75,13 @@ The -modcache flag causes clean to remove the entire module download cache, including unpacked source code of versioned dependencies. +The -fuzzcache flag causes clean to remove files stored in the Go build +cache for fuzz testing. The fuzzing engine caches files that expand +code coverage, so removing them may make fuzzing less effective until +new inputs are found that provide the same coverage. These files are +distinct from those stored in testdata directory; clean does not remove +those files. + For more about build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -85,6 +92,7 @@ var ( cleanI bool // clean -i flag cleanR bool // clean -r flag cleanCache bool // clean -cache flag + cleanFuzzcache bool // clean -fuzzcache flag cleanModcache bool // clean -modcache flag cleanTestcache bool // clean -testcache flag ) @@ -96,6 +104,7 @@ func init() { CmdClean.Flag.BoolVar(&cleanI, "i", false, "") CmdClean.Flag.BoolVar(&cleanR, "r", false, "") CmdClean.Flag.BoolVar(&cleanCache, "cache", false, "") + CmdClean.Flag.BoolVar(&cleanFuzzcache, "fuzzcache", false, "") CmdClean.Flag.BoolVar(&cleanModcache, "modcache", false, "") CmdClean.Flag.BoolVar(&cleanTestcache, "testcache", false, "") @@ -112,7 +121,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { // or no other target (such as a cache) was requested to be cleaned. cleanPkg := len(args) > 0 || cleanI || cleanR if (!modload.Enabled() || modload.HasModRoot()) && - !cleanCache && !cleanModcache && !cleanTestcache { + !cleanCache && !cleanModcache && !cleanTestcache && !cleanFuzzcache { cleanPkg = true } @@ -144,7 +153,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { // This also mimics what os.RemoveAll(dir) would do. if err := os.RemoveAll(d); err != nil && !printedErrors { printedErrors = true - base.Errorf("go clean -cache: %v", err) + base.Errorf("go: %v", err) } } } @@ -157,7 +166,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { if !cfg.BuildN { if err := os.RemoveAll(logFile); err != nil && !printedErrors { printedErrors = true - base.Errorf("go clean -cache: %v", err) + base.Errorf("go: %v", err) } } } @@ -187,7 +196,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { } if err != nil { if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { - base.Errorf("go clean -testcache: %v", err) + base.Errorf("go: %v", err) } } } @@ -195,14 +204,26 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { if cleanModcache { if cfg.GOMODCACHE == "" { - base.Fatalf("go clean -modcache: no module cache") + base.Fatalf("go: cannot clean -modcache without a module cache") } if cfg.BuildN || cfg.BuildX { b.Showcmd("", "rm -rf %s", cfg.GOMODCACHE) } if !cfg.BuildN { if err := modfetch.RemoveAll(cfg.GOMODCACHE); err != nil { - base.Errorf("go clean -modcache: %v", err) + base.Errorf("go: %v", err) + } + } + } + + if cleanFuzzcache { + fuzzDir := cache.Default().FuzzDir() + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "rm -rf %s", fuzzDir) + } + if !cfg.BuildN { + if err := os.RemoveAll(fuzzDir); err != nil { + base.Errorf("go: %v", err) } } } @@ -245,7 +266,7 @@ func clean(p *load.Package) { } dirs, err := os.ReadDir(p.Dir) if err != nil { - base.Errorf("go clean %s: %v", p.Dir, err) + base.Errorf("go: %s: %v", p.Dir, err) return } @@ -334,7 +355,7 @@ func clean(p *load.Package) { } } if err := os.RemoveAll(filepath.Join(p.Dir, name)); err != nil { - base.Errorf("go clean: %v", err) + base.Errorf("go: %v", err) } } continue @@ -386,5 +407,5 @@ func removeFile(f string) { return } } - base.Errorf("go clean: %v", err) + base.Errorf("go: %v", err) } diff --git a/libgo/go/cmd/go/internal/cmdflag/flag.go b/libgo/go/cmd/go/internal/cmdflag/flag.go index 8abb7e559f5..a634bc1ab8d 100644 --- a/libgo/go/cmd/go/internal/cmdflag/flag.go +++ b/libgo/go/cmd/go/internal/cmdflag/flag.go @@ -92,7 +92,7 @@ func ParseOne(fs *flag.FlagSet, args []string) (f *flag.Flag, remainingArgs []st // Use fs.Set instead of f.Value.Set below so that any subsequent call to // fs.Visit will correctly visit the flags that have been set. - failf := func(format string, a ...interface{}) (*flag.Flag, []string, error) { + failf := func(format string, a ...any) (*flag.Flag, []string, error) { return f, args, fmt.Errorf(format, a...) } diff --git a/libgo/go/cmd/go/internal/doc/doc.go b/libgo/go/cmd/go/internal/doc/doc.go index 8580a5dc4d2..7741a9022c9 100644 --- a/libgo/go/cmd/go/internal/doc/doc.go +++ b/libgo/go/cmd/go/internal/doc/doc.go @@ -60,9 +60,8 @@ The package path must be either a qualified path or a proper suffix of a path. The go tool's usual package mechanism does not apply: package path elements like . and ... are not implemented by go doc. -When run with two arguments, the first must be a full package path (not just a -suffix), and the second is a symbol, or symbol with method or struct field. -This is similar to the syntax accepted by godoc: +When run with two arguments, the first is a package path (full path or suffix), +and the second is a symbol, or symbol with method or struct field: go doc <pkg> <sym>[.<methodOrField>] diff --git a/libgo/go/cmd/go/internal/envcmd/env.go b/libgo/go/cmd/go/internal/envcmd/env.go index 1553d263914..e56dd8223f0 100644 --- a/libgo/go/cmd/go/internal/envcmd/env.go +++ b/libgo/go/cmd/go/internal/envcmd/env.go @@ -26,6 +26,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" + "cmd/internal/quoted" ) var CmdEnv = &base.Command{ @@ -104,13 +105,13 @@ func MkEnv() []cfg.EnvVar { env = append(env, cfg.EnvVar{Name: key, Value: val}) } - cc := cfg.DefaultCC(cfg.Goos, cfg.Goarch) - if env := strings.Fields(cfg.Getenv("CC")); len(env) > 0 { - cc = env[0] + cc := cfg.Getenv("CC") + if cc == "" { + cc = cfg.DefaultCC(cfg.Goos, cfg.Goarch) } - cxx := cfg.DefaultCXX(cfg.Goos, cfg.Goarch) - if env := strings.Fields(cfg.Getenv("CXX")); len(env) > 0 { - cxx = env[0] + cxx := cfg.Getenv("CXX") + if cxx == "" { + cxx = cfg.DefaultCXX(cfg.Goos, cfg.Goarch) } env = append(env, cfg.EnvVar{Name: "AR", Value: envOr("AR", "ar")}) env = append(env, cfg.EnvVar{Name: "CC", Value: cc}) @@ -145,13 +146,17 @@ func findEnv(env []cfg.EnvVar, name string) string { // ExtraEnvVars returns environment variables that should not leak into child processes. func ExtraEnvVars() []cfg.EnvVar { gomod := "" + modload.Init() if modload.HasModRoot() { - gomod = filepath.Join(modload.ModRoot(), "go.mod") + gomod = modload.ModFilePath() } else if modload.Enabled() { gomod = os.DevNull } + modload.InitWorkfile() + gowork := modload.WorkFilePath() return []cfg.EnvVar{ {Name: "GOMOD", Value: gomod}, + {Name: "GOWORK", Value: gowork}, } } @@ -191,13 +196,13 @@ func argKey(arg string) string { func runEnv(ctx context.Context, cmd *base.Command, args []string) { if *envJson && *envU { - base.Fatalf("go env: cannot use -json with -u") + base.Fatalf("go: cannot use -json with -u") } if *envJson && *envW { - base.Fatalf("go env: cannot use -json with -w") + base.Fatalf("go: cannot use -json with -w") } if *envU && *envW { - base.Fatalf("go env: cannot use -u with -w") + base.Fatalf("go: cannot use -u with -w") } // Handle 'go env -w' and 'go env -u' before calling buildcfg.Check, @@ -275,7 +280,7 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { func runEnvW(args []string) { // Process and sanity-check command line. if len(args) == 0 { - base.Fatalf("go env -w: no KEY=VALUE arguments given") + base.Fatalf("go: no KEY=VALUE arguments given") } osEnv := make(map[string]string) for _, e := range cfg.OrigEnv { @@ -287,14 +292,14 @@ func runEnvW(args []string) { for _, arg := range args { i := strings.Index(arg, "=") if i < 0 { - base.Fatalf("go env -w: arguments must be KEY=VALUE: invalid argument: %s", arg) + base.Fatalf("go: arguments must be KEY=VALUE: invalid argument: %s", arg) } key, val := arg[:i], arg[i+1:] if err := checkEnvWrite(key, val); err != nil { - base.Fatalf("go env -w: %v", err) + base.Fatalf("go: %v", err) } if _, ok := add[key]; ok { - base.Fatalf("go env -w: multiple values for key: %s", key) + base.Fatalf("go: multiple values for key: %s", key) } add[key] = val if osVal := osEnv[key]; osVal != "" && osVal != val { @@ -303,13 +308,13 @@ func runEnvW(args []string) { } if err := checkBuildConfig(add, nil); err != nil { - base.Fatalf("go env -w: %v", err) + base.Fatalf("go: %v", err) } gotmp, okGOTMP := add["GOTMPDIR"] if okGOTMP { if !filepath.IsAbs(gotmp) && gotmp != "" { - base.Fatalf("go env -w: GOTMPDIR must be an absolute path") + base.Fatalf("go: GOTMPDIR must be an absolute path") } } @@ -319,18 +324,18 @@ func runEnvW(args []string) { func runEnvU(args []string) { // Process and sanity-check command line. if len(args) == 0 { - base.Fatalf("go env -u: no arguments given") + base.Fatalf("go: 'go env -u' requires an argument") } del := make(map[string]bool) for _, arg := range args { if err := checkEnvWrite(arg, ""); err != nil { - base.Fatalf("go env -u: %v", err) + base.Fatalf("go: %v", err) } del[arg] = true } if err := checkBuildConfig(nil, del); err != nil { - base.Fatalf("go env -u: %v", err) + base.Fatalf("go: %v", err) } updateEnvFile(nil, del) @@ -414,7 +419,7 @@ func printEnvAsJSON(env []cfg.EnvVar) { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", "\t") if err := enc.Encode(m); err != nil { - base.Fatalf("go env -json: %s", err) + base.Fatalf("go: %s", err) } } @@ -429,7 +434,7 @@ func getOrigEnv(key string) string { func checkEnvWrite(key, val string) error { switch key { - case "GOEXE", "GOGCCFLAGS", "GOHOSTARCH", "GOHOSTOS", "GOMOD", "GOTOOLDIR", "GOVERSION": + case "GOEXE", "GOGCCFLAGS", "GOHOSTARCH", "GOHOSTOS", "GOMOD", "GOWORK", "GOTOOLDIR", "GOVERSION": return fmt.Errorf("%s cannot be modified", key) case "GOENV": return fmt.Errorf("%s can only be set using the OS environment", key) @@ -457,10 +462,23 @@ func checkEnvWrite(key, val string) error { if !filepath.IsAbs(val) && val != "" { return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val) } - // Make sure CC and CXX are absolute paths - case "CC", "CXX", "GOMODCACHE": - if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) { - return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val) + case "GOMODCACHE": + if !filepath.IsAbs(val) && val != "" { + return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q", val) + } + case "CC", "CXX": + if val == "" { + break + } + args, err := quoted.Split(val) + if err != nil { + return fmt.Errorf("invalid %s: %v", key, err) + } + if len(args) == 0 { + return fmt.Errorf("%s entry cannot contain only space", key) + } + if !filepath.IsAbs(args[0]) && args[0] != filepath.Base(args[0]) { + return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, args[0]) } } @@ -479,11 +497,11 @@ func checkEnvWrite(key, val string) error { func updateEnvFile(add map[string]string, del map[string]bool) { file, err := cfg.EnvFile() if file == "" { - base.Fatalf("go env: cannot find go env config: %v", err) + base.Fatalf("go: cannot find go env config: %v", err) } data, err := os.ReadFile(file) if err != nil && (!os.IsNotExist(err) || len(add) == 0) { - base.Fatalf("go env: reading go env config: %v", err) + base.Fatalf("go: reading go env config: %v", err) } lines := strings.SplitAfter(string(data), "\n") @@ -541,7 +559,7 @@ func updateEnvFile(add map[string]string, del map[string]bool) { os.MkdirAll(filepath.Dir(file), 0777) err = os.WriteFile(file, data, 0666) if err != nil { - base.Fatalf("go env: writing go env config: %v", err) + base.Fatalf("go: writing go env config: %v", err) } } } diff --git a/libgo/go/cmd/go/internal/fix/fix.go b/libgo/go/cmd/go/internal/fix/fix.go index 988d45e71cc..d8ba353de65 100644 --- a/libgo/go/cmd/go/internal/fix/fix.go +++ b/libgo/go/cmd/go/internal/fix/fix.go @@ -11,27 +11,39 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/str" + "cmd/go/internal/work" "context" "fmt" + "go/build" "os" ) var CmdFix = &base.Command{ - Run: runFix, - UsageLine: "go fix [packages]", + UsageLine: "go fix [-fix list] [packages]", Short: "update packages to use new APIs", Long: ` Fix runs the Go fix command on the packages named by the import paths. +The -fix flag sets a comma-separated list of fixes to run. +The default is all known fixes. +(Its value is passed to 'go tool fix -r'.) + For more about fix, see 'go doc cmd/fix'. For more about specifying packages, see 'go help packages'. -To run fix with specific options, run 'go tool fix'. +To run fix with other options, run 'go tool fix'. See also: go fmt, go vet. `, } +var fixes = CmdFix.Flag.String("fix", "", "comma-separated list of fixes to apply") + +func init() { + work.AddBuildFlags(CmdFix, work.DefaultBuildFlags) + CmdFix.Run = runFix // fix cycle +} + func runFix(ctx context.Context, cmd *base.Command, args []string) { pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{}, args) w := 0 @@ -58,6 +70,16 @@ func runFix(ctx context.Context, cmd *base.Command, args []string) { // the command only applies to this package, // not to packages in subdirectories. files := base.RelPaths(pkg.InternalAllGoFiles()) - base.Run(str.StringList(cfg.BuildToolexec, base.Tool("fix"), files)) + goVersion := "" + if pkg.Module != nil { + goVersion = "go" + pkg.Module.GoVersion + } else if pkg.Standard { + goVersion = build.Default.ReleaseTags[len(build.Default.ReleaseTags)-1] + } + var fixArg []string + if *fixes != "" { + fixArg = []string{"-r=" + *fixes} + } + base.Run(str.StringList(cfg.BuildToolexec, base.Tool("fix"), "-go="+goVersion, fixArg, files)) } } diff --git a/libgo/go/cmd/go/internal/fmtcmd/fmt.go b/libgo/go/cmd/go/internal/fmtcmd/fmt.go index 8a040087539..19656eab7fc 100644 --- a/libgo/go/cmd/go/internal/fmtcmd/fmt.go +++ b/libgo/go/cmd/go/internal/fmtcmd/fmt.go @@ -11,14 +11,12 @@ import ( "fmt" "os" "path/filepath" - "runtime" - "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" + "cmd/internal/sys" ) func init() { @@ -53,18 +51,13 @@ See also: go fix, go vet. func runFmt(ctx context.Context, cmd *base.Command, args []string) { printed := false gofmt := gofmtPath() - procs := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - wg.Add(procs) - fileC := make(chan string, 2*procs) - for i := 0; i < procs; i++ { - go func() { - defer wg.Done() - for file := range fileC { - base.Run(str.StringList(gofmt, "-l", "-w", file)) - } - }() - } + + gofmtArgs := []string{gofmt, "-l", "-w"} + gofmtArgLen := len(gofmt) + len(" -l -w") + + baseGofmtArgs := len(gofmtArgs) + baseGofmtArgLen := gofmtArgLen + for _, pkg := range load.PackagesAndErrors(ctx, load.PackageOpts{}, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { @@ -89,11 +82,18 @@ func runFmt(ctx context.Context, cmd *base.Command, args []string) { // not to packages in subdirectories. files := base.RelPaths(pkg.InternalAllGoFiles()) for _, file := range files { - fileC <- file + gofmtArgs = append(gofmtArgs, file) + gofmtArgLen += 1 + len(file) // plus separator + if gofmtArgLen >= sys.ExecArgLengthLimit { + base.Run(gofmtArgs) + gofmtArgs = gofmtArgs[:baseGofmtArgs] + gofmtArgLen = baseGofmtArgLen + } } } - close(fileC) - wg.Wait() + if len(gofmtArgs) > baseGofmtArgs { + base.Run(gofmtArgs) + } } func gofmtPath() string { diff --git a/libgo/go/cmd/go/internal/fsys/fsys.go b/libgo/go/cmd/go/internal/fsys/fsys.go index 0b806027e64..9a1bbf890e1 100644 --- a/libgo/go/cmd/go/internal/fsys/fsys.go +++ b/libgo/go/cmd/go/internal/fsys/fsys.go @@ -499,7 +499,7 @@ func (f fakeFile) Size() int64 { return f.real.Size() } func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } func (f fakeFile) IsDir() bool { return f.real.IsDir() } -func (f fakeFile) Sys() interface{} { return f.real.Sys() } +func (f fakeFile) Sys() any { return f.real.Sys() } // missingFile provides an fs.FileInfo for an overlaid file where the // destination file in the overlay doesn't exist. It returns zero values @@ -512,7 +512,7 @@ func (f missingFile) Size() int64 { return 0 } func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } func (f missingFile) IsDir() bool { return false } -func (f missingFile) Sys() interface{} { return nil } +func (f missingFile) Sys() any { return nil } // fakeDir provides an fs.FileInfo implementation for directories that are // implicitly created by overlaid files. Each directory in the @@ -524,7 +524,7 @@ func (f fakeDir) Size() int64 { return 0 } func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } func (f fakeDir) IsDir() bool { return true } -func (f fakeDir) Sys() interface{} { return nil } +func (f fakeDir) Sys() any { return nil } // Glob is like filepath.Glob but uses the overlay file system. func Glob(pattern string) (matches []string, err error) { diff --git a/libgo/go/cmd/go/internal/fsys/fsys_test.go b/libgo/go/cmd/go/internal/fsys/fsys_test.go index 7f175c70311..c080c14987c 100644 --- a/libgo/go/cmd/go/internal/fsys/fsys_test.go +++ b/libgo/go/cmd/go/internal/fsys/fsys_test.go @@ -1,7 +1,6 @@ package fsys import ( - "cmd/go/internal/txtar" "encoding/json" "errors" "fmt" @@ -12,6 +11,8 @@ import ( "path/filepath" "reflect" "testing" + + "golang.org/x/tools/txtar" ) // initOverlay resets the overlay state to reflect the config. diff --git a/libgo/go/cmd/go/internal/generate/generate.go b/libgo/go/cmd/go/internal/generate/generate.go index 80ea32b4284..54ccfe78f24 100644 --- a/libgo/go/cmd/go/internal/generate/generate.go +++ b/libgo/go/cmd/go/internal/generate/generate.go @@ -38,7 +38,7 @@ Generate runs commands described by directives within existing files. Those commands can run any process but the intent is to create or update Go source files. -Go generate is never run automatically by go build, go get, go test, +Go generate is never run automatically by go build, go test, and so on. It must be run explicitly. Go generate scans the file for directives, which are lines of @@ -408,7 +408,7 @@ var stop = fmt.Errorf("error in generation") // errorf logs an error message prefixed with the file and line number. // It then exits the program (with exit status 1) because generation stops // at the first error. -func (g *Generator) errorf(format string, args ...interface{}) { +func (g *Generator) errorf(format string, args ...any) { fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum, fmt.Sprintf(format, args...)) panic(stop) diff --git a/libgo/go/cmd/go/internal/get/get.go b/libgo/go/cmd/go/internal/get/get.go index 3c16dc3040f..8cf8fe6645f 100644 --- a/libgo/go/cmd/go/internal/get/get.go +++ b/libgo/go/cmd/go/internal/get/get.go @@ -114,16 +114,16 @@ func init() { func runGet(ctx context.Context, cmd *base.Command, args []string) { if cfg.ModulesEnabled { // Should not happen: main.go should install the separate module-enabled get code. - base.Fatalf("go get: modules not implemented") + base.Fatalf("go: modules not implemented") } work.BuildInit() if *getF && !*getU { - base.Fatalf("go get: cannot use -f flag without -u") + base.Fatalf("go: cannot use -f flag without -u") } if *getInsecure { - base.Fatalf("go get: -insecure flag is no longer supported; use GOINSECURE instead") + base.Fatalf("go: -insecure flag is no longer supported; use GOINSECURE instead") } // Disable any prompting for passwords by Git itself. @@ -214,18 +214,19 @@ func downloadPaths(patterns []string) []string { // if the argument has no slash or refers to an existing file. if strings.HasSuffix(arg, ".go") { if !strings.Contains(arg, "/") { - base.Errorf("go get %s: arguments must be package or module paths", arg) + base.Errorf("go: %s: arguments must be package or module paths", arg) continue } if fi, err := os.Stat(arg); err == nil && !fi.IsDir() { - base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", arg) + base.Errorf("go: %s exists as a file, but 'go get' requires package arguments", arg) } } } base.ExitIfErrors() var pkgs []string - for _, m := range search.ImportPathsQuiet(patterns) { + noModRoots := []string{} + for _, m := range search.ImportPathsQuiet(patterns, noModRoots) { if len(m.Pkgs) == 0 && strings.Contains(m.Pattern(), "...") { pkgs = append(pkgs, m.Pattern()) } else { @@ -315,7 +316,8 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) if wildcardOkay && strings.Contains(arg, "...") { match := search.NewMatch(arg) if match.IsLocal() { - match.MatchDirs() + noModRoots := []string{} // We're in gopath mode, so there are no modroots. + match.MatchDirs(noModRoots) args = match.Dirs } else { match.MatchPackages() @@ -415,10 +417,10 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // to make the first copy of or update a copy of the given package. func downloadPackage(p *load.Package) error { var ( - vcsCmd *vcs.Cmd - repo, rootPath string - err error - blindRepo bool // set if the repo has unusual configuration + vcsCmd *vcs.Cmd + repo, rootPath, repoDir string + err error + blindRepo bool // set if the repo has unusual configuration ) // p can be either a real package, or a pseudo-package whose “import path” is @@ -444,10 +446,19 @@ func downloadPackage(p *load.Package) error { if p.Internal.Build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. - vcsCmd, rootPath, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot) + const allowNesting = false + repoDir, vcsCmd, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot, allowNesting) if err != nil { return err } + if !str.HasFilePathPrefix(repoDir, p.Internal.Build.SrcRoot) { + panic(fmt.Sprintf("repository %q not in source root %q", repo, p.Internal.Build.SrcRoot)) + } + rootPath = str.TrimFilePathPrefix(repoDir, p.Internal.Build.SrcRoot) + if err := vcs.CheckGOVCS(vcsCmd, rootPath); err != nil { + return err + } + repo = "<local>" // should be unused; make distinctive // Double-check where it came from. diff --git a/libgo/go/cmd/go/internal/help/help.go b/libgo/go/cmd/go/internal/help/help.go index 7a730fc8eb8..2a07d2423bd 100644 --- a/libgo/go/cmd/go/internal/help/help.go +++ b/libgo/go/cmd/go/internal/help/help.go @@ -162,7 +162,7 @@ func (w *errWriter) Write(b []byte) (int, error) { } // tmpl executes the given template text on data, writing the result to w. -func tmpl(w io.Writer, text string, data interface{}) { +func tmpl(w io.Writer, text string, data any) { t := template.New("top") t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) template.Must(t.Parse(text)) diff --git a/libgo/go/cmd/go/internal/help/helpdoc.go b/libgo/go/cmd/go/internal/help/helpdoc.go index 490ff1fb7cf..7dc066cfbad 100644 --- a/libgo/go/cmd/go/internal/help/helpdoc.go +++ b/libgo/go/cmd/go/internal/help/helpdoc.go @@ -592,6 +592,10 @@ Architecture-specific environment variables: GO386 For GOARCH=386, how to implement floating point instructions. Valid values are sse2 (default), softfloat. + GOAMD64 + For GOARCH=amd64, the microarchitecture level for which to compile. + Valid values are v1 (default), v2, v3, v4. + See https://golang.org/wiki/MinimumRequirements#amd64 GOMIPS For GOARCH=mips{,le}, whether to use floating point instructions. Valid values are hardfloat (default), softfloat. @@ -771,6 +775,13 @@ The go command also caches successful package test results. See 'go help test' for details. Running 'go clean -testcache' removes all cached test results (but not cached build results). +The go command also caches values used in fuzzing with 'go test -fuzz', +specifically, values that expanded code coverage when passed to a +fuzz function. These values are not used for regular building and +testing, but they're stored in a subdirectory of the build cache. +Running 'go clean -fuzzcache' removes all cached fuzzing values. +This may make fuzzing less effective, temporarily. + The GODEBUG environment variable can enable printing of debugging information about the state of the cache: diff --git a/libgo/go/cmd/go/internal/imports/build.go b/libgo/go/cmd/go/internal/imports/build.go index 50aeabc578c..ff6bea6777d 100644 --- a/libgo/go/cmd/go/internal/imports/build.go +++ b/libgo/go/cmd/go/internal/imports/build.go @@ -2,17 +2,51 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Copied from Go distribution src/go/build/build.go, syslist.go +// Copied from Go distribution src/go/build/build.go, syslist.go. +// That package does not export the ability to process raw file data, +// although we could fake it with an appropriate build.Context +// and a lot of unwrapping. +// More importantly, that package does not implement the tags["*"] +// special case, in which both tag and !tag are considered to be true +// for essentially all tags (except "ignore"). +// +// If we added this API to go/build directly, we wouldn't need this +// file anymore, but this API is not terribly general-purpose and we +// don't really want to commit to any public form of it, nor do we +// want to move the core parts of go/build into a top-level internal package. +// These details change very infrequently, so the copy is fine. package imports import ( "bytes" + "errors" + "fmt" + "go/build/constraint" "strings" "unicode" ) -var slashslash = []byte("//") +var ( + bSlashSlash = []byte("//") + bStarSlash = []byte("*/") + bSlashStar = []byte("/*") + bPlusBuild = []byte("+build") + + goBuildComment = []byte("//go:build") + + errGoBuildWithoutBuild = errors.New("//go:build comment without // +build comment") + errMultipleGoBuild = errors.New("multiple //go:build comments") +) + +func isGoBuildComment(line []byte) bool { + if !bytes.HasPrefix(line, goBuildComment) { + return false + } + line = bytes.TrimSpace(line) + rest := line[len(goBuildComment):] + return len(rest) == 0 || len(bytes.TrimSpace(rest)) < len(rest) +} // ShouldBuild reports whether it is okay to use this file, // The rule is that in the file's leading run of // comments @@ -34,10 +68,61 @@ var slashslash = []byte("//") // in any build. // func ShouldBuild(content []byte, tags map[string]bool) bool { - // Pass 1. Identify leading run of // comments and blank lines, + // Identify leading run of // comments and blank lines, // which must be followed by a blank line. + // Also identify any //go:build comments. + content, goBuild, _, err := parseFileHeader(content) + if err != nil { + return false + } + + // If //go:build line is present, it controls. + // Otherwise fall back to +build processing. + var shouldBuild bool + switch { + case goBuild != nil: + x, err := constraint.Parse(string(goBuild)) + if err != nil { + return false + } + shouldBuild = eval(x, tags, true) + + default: + shouldBuild = true + p := content + for len(p) > 0 { + line := p + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, p = line[:i], p[i+1:] + } else { + p = p[len(p):] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, bSlashSlash) || !bytes.Contains(line, bPlusBuild) { + continue + } + text := string(line) + if !constraint.IsPlusBuild(text) { + continue + } + if x, err := constraint.Parse(text); err == nil { + if !eval(x, tags, true) { + shouldBuild = false + } + } + } + } + + return shouldBuild +} + +func parseFileHeader(content []byte) (trimmed, goBuild []byte, sawBinaryOnly bool, err error) { end := 0 p := content + ended := false // found non-blank, non-// line, so stopped accepting // +build lines + inSlashStar := false // in /* */ comment + +Lines: for len(p) > 0 { line := p if i := bytes.IndexByte(line, '\n'); i >= 0 { @@ -46,78 +131,61 @@ func ShouldBuild(content []byte, tags map[string]bool) bool { p = p[len(p):] } line = bytes.TrimSpace(line) - if len(line) == 0 { // Blank line + if len(line) == 0 && !ended { // Blank line + // Remember position of most recent blank line. + // When we find the first non-blank, non-// line, + // this "end" position marks the latest file position + // where a // +build line can appear. + // (It must appear _before_ a blank line before the non-blank, non-// line. + // Yes, that's confusing, which is part of why we moved to //go:build lines.) + // Note that ended==false here means that inSlashStar==false, + // since seeing a /* would have set ended==true. end = len(content) - len(p) - continue + continue Lines } - if !bytes.HasPrefix(line, slashslash) { // Not comment line - break + if !bytes.HasPrefix(line, bSlashSlash) { // Not comment line + ended = true } - } - content = content[:end] - // Pass 2. Process each line in the run. - p = content - allok := true - for len(p) > 0 { - line := p - if i := bytes.IndexByte(line, '\n'); i >= 0 { - line, p = line[:i], p[i+1:] - } else { - p = p[len(p):] - } - line = bytes.TrimSpace(line) - if !bytes.HasPrefix(line, slashslash) { - continue + if !inSlashStar && isGoBuildComment(line) { + if goBuild != nil { + return nil, nil, false, errMultipleGoBuild + } + goBuild = line } - line = bytes.TrimSpace(line[len(slashslash):]) - if len(line) > 0 && line[0] == '+' { - // Looks like a comment +line. - f := strings.Fields(string(line)) - if f[0] == "+build" { - ok := false - for _, tok := range f[1:] { - if matchTags(tok, tags) { - ok = true - } - } - if !ok { - allok = false + + Comments: + for len(line) > 0 { + if inSlashStar { + if i := bytes.Index(line, bStarSlash); i >= 0 { + inSlashStar = false + line = bytes.TrimSpace(line[i+len(bStarSlash):]) + continue Comments } + continue Lines } + if bytes.HasPrefix(line, bSlashSlash) { + continue Lines + } + if bytes.HasPrefix(line, bSlashStar) { + inSlashStar = true + line = bytes.TrimSpace(line[len(bSlashStar):]) + continue Comments + } + // Found non-comment text. + break Lines } } - return allok -} - -// matchTags reports whether the name is one of: -// -// tag (if tags[tag] is true) -// !tag (if tags[tag] is false) -// a comma-separated list of any of these -// -func matchTags(name string, tags map[string]bool) bool { - if name == "" { - return false - } - if i := strings.Index(name, ","); i >= 0 { - // comma-separated list - ok1 := matchTags(name[:i], tags) - ok2 := matchTags(name[i+1:], tags) - return ok1 && ok2 - } - if strings.HasPrefix(name, "!!") { // bad syntax, reject always - return false - } - if strings.HasPrefix(name, "!") { // negation - return len(name) > 1 && matchTag(name[1:], tags, false) - } - return matchTag(name, tags, true) + return content[:end], goBuild, sawBinaryOnly, nil } -// matchTag reports whether the tag name is valid and satisfied by tags[name]==want. -func matchTag(name string, tags map[string]bool, want bool) bool { +// matchTag reports whether the tag name is valid and tags[name] is true. +// As a special case, if tags["*"] is true and name is not empty or ignore, +// then matchTag will return prefer instead of the actual answer, +// which allows the caller to pretend in that case that most tags are +// both true and false. +func matchTag(name string, tags map[string]bool, prefer bool) bool { // Tags must be letters, digits, underscores or dots. // Unlike in Go identifiers, all digits are fine (e.g., "386"). for _, c := range name { @@ -131,7 +199,7 @@ func matchTag(name string, tags map[string]bool, want bool) bool { // if we put * in the tags map then all tags // except "ignore" are considered both present and not // (so we return true no matter how 'want' is set). - return true + return prefer } have := tags[name] @@ -144,7 +212,25 @@ func matchTag(name string, tags map[string]bool, want bool) bool { if name == "darwin" { have = have || tags["ios"] } - return have == want + return have +} + +// eval is like +// x.Eval(func(tag string) bool { return matchTag(tag, tags) }) +// except that it implements the special case for tags["*"] meaning +// all tags are both true and false at the same time. +func eval(x constraint.Expr, tags map[string]bool, prefer bool) bool { + switch x := x.(type) { + case *constraint.TagExpr: + return matchTag(x.Tag, tags, prefer) + case *constraint.NotExpr: + return !eval(x.X, tags, !prefer) + case *constraint.AndExpr: + return eval(x.X, tags, prefer) && eval(x.Y, tags, prefer) + case *constraint.OrExpr: + return eval(x.X, tags, prefer) || eval(x.Y, tags, prefer) + } + panic(fmt.Sprintf("unexpected constraint expression %T", x)) } // MatchFile returns false if the name contains a $GOOS or $GOARCH diff --git a/libgo/go/cmd/go/internal/imports/scan_test.go b/libgo/go/cmd/go/internal/imports/scan_test.go index 2d245ee7872..7e69c56513a 100644 --- a/libgo/go/cmd/go/internal/imports/scan_test.go +++ b/libgo/go/cmd/go/internal/imports/scan_test.go @@ -33,7 +33,7 @@ func TestScan(t *testing.T) { } if p == "net/http" { // A test import but not an import - t.Errorf("json reported as importing encoding/binary but does not") + t.Errorf("json reported as importing net/http but does not") } } if !foundBase64 { diff --git a/libgo/go/cmd/go/internal/imports/testdata/android/e.go b/libgo/go/cmd/go/internal/imports/testdata/android/e.go index d9b2db769b5..f1b9c888c2c 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/android/e.go +++ b/libgo/go/cmd/go/internal/imports/testdata/android/e.go @@ -1,3 +1,4 @@ +//go:build android // +build android package android diff --git a/libgo/go/cmd/go/internal/imports/testdata/android/f.go b/libgo/go/cmd/go/internal/imports/testdata/android/f.go index 281e4dd6b98..bb0ff7b73f6 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/android/f.go +++ b/libgo/go/cmd/go/internal/imports/testdata/android/f.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package android diff --git a/libgo/go/cmd/go/internal/imports/testdata/android/g.go b/libgo/go/cmd/go/internal/imports/testdata/android/g.go index 66a789c0ada..ee19424890a 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/android/g.go +++ b/libgo/go/cmd/go/internal/imports/testdata/android/g.go @@ -1,3 +1,4 @@ +//go:build !android // +build !android package android diff --git a/libgo/go/cmd/go/internal/imports/testdata/illumos/e.go b/libgo/go/cmd/go/internal/imports/testdata/illumos/e.go index 5e1ed3cb9de..fddf2c42990 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/illumos/e.go +++ b/libgo/go/cmd/go/internal/imports/testdata/illumos/e.go @@ -1,3 +1,4 @@ +//go:build illumos // +build illumos package illumos diff --git a/libgo/go/cmd/go/internal/imports/testdata/illumos/f.go b/libgo/go/cmd/go/internal/imports/testdata/illumos/f.go index f3e3f728bce..4b6d528e4c2 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/illumos/f.go +++ b/libgo/go/cmd/go/internal/imports/testdata/illumos/f.go @@ -1,3 +1,4 @@ +//go:build solaris // +build solaris package illumos diff --git a/libgo/go/cmd/go/internal/imports/testdata/illumos/g.go b/libgo/go/cmd/go/internal/imports/testdata/illumos/g.go index b30f1eb4037..1bf826b8151 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/illumos/g.go +++ b/libgo/go/cmd/go/internal/imports/testdata/illumos/g.go @@ -1,3 +1,4 @@ +//go:build !illumos // +build !illumos package illumos diff --git a/libgo/go/cmd/go/internal/imports/testdata/star/x1.go b/libgo/go/cmd/go/internal/imports/testdata/star/x1.go index 6a9594aed03..eaaea979e9d 100644 --- a/libgo/go/cmd/go/internal/imports/testdata/star/x1.go +++ b/libgo/go/cmd/go/internal/imports/testdata/star/x1.go @@ -1,8 +1,5 @@ -// +build blahblh -// +build linux -// +build !linux -// +build windows -// +build darwin +//go:build blahblh && linux && !linux && windows && darwin +// +build blahblh,linux,!linux,windows,darwin package x diff --git a/libgo/go/cmd/go/internal/list/list.go b/libgo/go/cmd/go/internal/list/list.go index 7cb9ec6d949..d9a7078ccf2 100644 --- a/libgo/go/cmd/go/internal/list/list.go +++ b/libgo/go/cmd/go/internal/list/list.go @@ -316,6 +316,7 @@ For more about modules, see https://golang.org/ref/mod. func init() { CmdList.Run = runList // break init cycle work.AddBuildFlags(CmdList, work.DefaultBuildFlags) + base.AddWorkfileFlag(&CmdList.Flag) } var ( @@ -336,6 +337,8 @@ var ( var nl = []byte{'\n'} func runList(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if *listFmt != "" && *listJson == true { base.Fatalf("go list -f cannot be used with -json") } @@ -355,9 +358,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } - var do func(interface{}) + var do func(any) if *listJson { - do = func(x interface{}) { + do = func(x any) { b, err := json.MarshalIndent(x, "", "\t") if err != nil { out.Flush() @@ -383,7 +386,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { if err != nil { base.Fatalf("%s", err) } - do = func(x interface{}) { + do = func(x any) { if err := tmpl.Execute(out, x); err != nil { out.Flush() base.Fatalf("%s", err) @@ -424,12 +427,12 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } if modload.Init(); !modload.Enabled() { - base.Fatalf("go list -m: not using modules") + base.Fatalf("go: list -m cannot be used with GO111MODULE=off") } modload.LoadModFile(ctx) // Sets cfg.BuildMod as a side-effect. if cfg.BuildMod == "vendor" { - const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" + const actionDisabledFormat = "go: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" if *listVersions { base.Fatalf(actionDisabledFormat, "determine available versions") @@ -468,11 +471,11 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { if !*listE { for _, m := range mods { if m.Error != nil { - base.Errorf("go list -m: %v", m.Error.Err) + base.Errorf("go: %v", m.Error.Err) } } if err != nil { - base.Errorf("go list -m: %v", err) + base.Errorf("go: %v", err) } base.ExitIfErrors() } @@ -708,7 +711,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } rmods, err := modload.ListModules(ctx, args, mode) if err != nil && !*listE { - base.Errorf("go list -retracted: %v", err) + base.Errorf("go: %v", err) } for i, arg := range args { rmod := rmods[i] diff --git a/libgo/go/cmd/go/internal/load/flag.go b/libgo/go/cmd/go/internal/load/flag.go index 440cb861344..de079decdf2 100644 --- a/libgo/go/cmd/go/internal/load/flag.go +++ b/libgo/go/cmd/go/internal/load/flag.go @@ -6,7 +6,7 @@ package load import ( "cmd/go/internal/base" - "cmd/go/internal/str" + "cmd/internal/quoted" "fmt" "strings" ) @@ -22,6 +22,7 @@ var ( // that allows specifying different effective flags for different packages. // See 'go help build' for more details about per-package flags. type PerPackageFlag struct { + raw string present bool values []ppfValue } @@ -39,6 +40,7 @@ func (f *PerPackageFlag) Set(v string) error { // set is the implementation of Set, taking a cwd (current working directory) for easier testing. func (f *PerPackageFlag) set(v, cwd string) error { + f.raw = v f.present = true match := func(p *Package) bool { return p.Internal.CmdlinePkg || p.Internal.CmdlineFiles } // default predicate with no pattern // For backwards compatibility with earlier flag splitting, ignore spaces around flags. @@ -61,7 +63,7 @@ func (f *PerPackageFlag) set(v, cwd string) error { match = MatchPackage(pattern, cwd) v = v[i+1:] } - flags, err := str.SplitQuotedFields(v) + flags, err := quoted.Split(v) if err != nil { return err } @@ -72,9 +74,7 @@ func (f *PerPackageFlag) set(v, cwd string) error { return nil } -// String is required to implement flag.Value. -// It is not used, because cmd/go never calls flag.PrintDefaults. -func (f *PerPackageFlag) String() string { return "<PerPackageFlag>" } +func (f *PerPackageFlag) String() string { return f.raw } // Present reports whether the flag appeared on the command line. func (f *PerPackageFlag) Present() bool { diff --git a/libgo/go/cmd/go/internal/load/pkg.go b/libgo/go/cmd/go/internal/load/pkg.go index c085fcbbf80..67b359d19d2 100644 --- a/libgo/go/cmd/go/internal/load/pkg.go +++ b/libgo/go/cmd/go/internal/load/pkg.go @@ -21,9 +21,11 @@ import ( pathpkg "path" "path/filepath" "runtime" + "runtime/debug" "sort" "strconv" "strings" + "time" "unicode" "unicode/utf8" @@ -38,6 +40,7 @@ import ( "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/go/internal/vcs" "cmd/internal/sys" "golang.org/x/mod/modfile" @@ -203,6 +206,7 @@ type PackageInternal struct { Local bool // imported via local path (./ or ../) LocalPrefix string // interpret ./ and ../ imports relative to this prefix ExeName string // desired name for temporary executable + FuzzInstrument bool // package should be instrumented for fuzzing CoverMode string // preprocess Go source files with the coverage tool in this mode CoverVars map[string]*CoverVar // variables created by coverage analysis OmitDebug bool // tell linker not to write debug information @@ -494,7 +498,7 @@ type importError struct { err error // created with fmt.Errorf } -func ImportErrorf(path, format string, args ...interface{}) ImportPathError { +func ImportErrorf(path, format string, args ...any) ImportPathError { err := &importError{importPath: path, err: fmt.Errorf(format, args...)} if errStr := err.Error(); !strings.Contains(errStr, path) { panic(fmt.Sprintf("path %q not in error %q", path, errStr)) @@ -585,10 +589,10 @@ func ClearPackageCachePartial(args []string) { delete(packageCache, arg) } } - resolvedImportCache.DeleteIf(func(key interface{}) bool { + resolvedImportCache.DeleteIf(func(key any) bool { return shouldDelete[key.(importSpec).path] }) - packageDataCache.DeleteIf(func(key interface{}) bool { + packageDataCache.DeleteIf(func(key any) bool { return shouldDelete[key.(string)] }) } @@ -601,7 +605,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { p := packageCache[arg] if p != nil { delete(packageCache, arg) - resolvedImportCache.DeleteIf(func(key interface{}) bool { + resolvedImportCache.DeleteIf(func(key any) bool { return key.(importSpec).path == p.ImportPath }) packageDataCache.Delete(p.ImportPath) @@ -813,7 +817,7 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo parentIsStd: parentIsStd, mode: mode, } - r := resolvedImportCache.Do(importKey, func() interface{} { + r := resolvedImportCache.Do(importKey, func() any { var r resolvedImport if build.IsLocalImport(path) { r.dir = filepath.Join(parentDir, path) @@ -840,7 +844,7 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo // Load the package from its directory. If we already found the package's // directory when resolving its import path, use that. - data := packageDataCache.Do(r.path, func() interface{} { + data := packageDataCache.Do(r.path, func() any { loaded = true var data packageData if r.dir != "" { @@ -1059,7 +1063,7 @@ func cleanImport(path string) string { var isDirCache par.Cache func isDir(path string) bool { - return isDirCache.Do(path, func() interface{} { + return isDirCache.Do(path, func() any { fi, err := fsys.Stat(path) return err == nil && fi.IsDir() }).(bool) @@ -1187,7 +1191,7 @@ var ( // goModPath returns the module path in the go.mod in dir, if any. func goModPath(dir string) (path string) { - return goModPathCache.Do(dir, func() interface{} { + return goModPathCache.Do(dir, func() any { data, err := os.ReadFile(filepath.Join(dir, "go.mod")) if err != nil { return "" @@ -1456,9 +1460,9 @@ func disallowInternal(ctx context.Context, srcDir string, importer *Package, imp // The importer is a list of command-line files. // Pretend that the import path is the import path of the // directory containing them. - // If the directory is outside the main module, this will resolve to ".", + // If the directory is outside the main modules, this will resolve to ".", // which is not a prefix of any valid module. - importerPath = modload.DirImportPath(ctx, importer.Dir) + importerPath, _ = modload.MainModules.DirImportPath(ctx, importer.Dir) } parentOfInternal := p.ImportPath[:i] if str.HasPathPrefix(importerPath, parentOfInternal) { @@ -1628,6 +1632,7 @@ var cgoSyscallExclude = map[string]bool{ "runtime/cgo": true, "runtime/race": true, "runtime/msan": true, + "runtime/asan": true, } var foldPath = make(map[string]string) @@ -1683,9 +1688,10 @@ func (p *Package) DefaultExecName() string { func (p *Package) load(ctx context.Context, opts PackageOpts, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(opts, bp) - // The localPrefix is the path we interpret ./ imports relative to. + // The localPrefix is the path we interpret ./ imports relative to, + // if we support them at all (not in module mode!). // Synthesized main packages sometimes override this. - if p.Internal.Local { + if p.Internal.Local && !cfg.ModulesEnabled { p.Internal.LocalPrefix = dirToImportPath(p.Dir) } @@ -1925,9 +1931,8 @@ func (p *Package) load(ctx context.Context, opts PackageOpts, path string, stk * } p.Internal.Imports = imports p.collectDeps() - - if cfg.ModulesEnabled && p.Error == nil && p.Name == "main" && len(p.DepsErrors) == 0 { - p.Internal.BuildInfo = modload.PackageBuildInfo(pkgPath, p.Deps) + if p.Error == nil && p.Name == "main" && len(p.DepsErrors) == 0 { + p.setBuildInfo() } // unsafe is a fake package. @@ -2018,13 +2023,18 @@ func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[st for _, pattern = range patterns { pid++ + glob := pattern + all := strings.HasPrefix(pattern, "all:") + if all { + glob = pattern[len("all:"):] + } // Check pattern is valid for //go:embed. - if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) { + if _, err := path.Match(glob, ""); err != nil || !validEmbedPattern(glob) { return nil, nil, fmt.Errorf("invalid pattern syntax") } // Glob to find matches. - match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(pattern)) + match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(glob)) if err != nil { return nil, nil, err } @@ -2087,7 +2097,7 @@ func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[st } rel := filepath.ToSlash(path[len(pkgdir)+1:]) name := info.Name() - if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') { + if path != file && (isBadEmbedName(name) || ((name[0] == '.' || name[0] == '_') && !all)) { // Ignore bad names, assuming they won't go into modules. // Also avoid hidden files that user may not know about. // See golang.org/issue/42328. @@ -2199,6 +2209,229 @@ func (p *Package) collectDeps() { } } +// vcsStatusCache maps repository directories (string) +// to their VCS information (vcsStatusError). +var vcsStatusCache par.Cache + +// setBuildInfo gathers build information, formats it as a string to be +// embedded in the binary, then sets p.Internal.BuildInfo to that string. +// setBuildInfo should only be called on a main package with no errors. +// +// This information can be retrieved using debug.ReadBuildInfo. +// +// Note that the GoVersion field is not set here to avoid encoding it twice. +// It is stored separately in the binary, mostly for historical reasons. +func (p *Package) setBuildInfo() { + // TODO: build and vcs information is not embedded for executables in GOROOT. + // cmd/dist uses -gcflags=all= -ldflags=all= by default, which means these + // executables always appear stale unless the user sets the same flags. + // Perhaps it's safe to omit those flags when GO_GCFLAGS and GO_LDFLAGS + // are not set? + setPkgErrorf := func(format string, args ...any) { + if p.Error == nil { + p.Error = &PackageError{Err: fmt.Errorf(format, args...)} + } + } + + var debugModFromModinfo func(*modinfo.ModulePublic) *debug.Module + debugModFromModinfo = func(mi *modinfo.ModulePublic) *debug.Module { + dm := &debug.Module{ + Path: mi.Path, + Version: mi.Version, + } + if mi.Replace != nil { + dm.Replace = debugModFromModinfo(mi.Replace) + } else { + dm.Sum = modfetch.Sum(module.Version{Path: mi.Path, Version: mi.Version}) + } + return dm + } + + var main debug.Module + if p.Module != nil { + main = *debugModFromModinfo(p.Module) + } + + visited := make(map[*Package]bool) + mdeps := make(map[module.Version]*debug.Module) + var q []*Package + q = append(q, p.Internal.Imports...) + for len(q) > 0 { + p1 := q[0] + q = q[1:] + if visited[p1] { + continue + } + visited[p1] = true + if p1.Module != nil { + m := module.Version{Path: p1.Module.Path, Version: p1.Module.Version} + if p1.Module.Path != main.Path && mdeps[m] == nil { + mdeps[m] = debugModFromModinfo(p1.Module) + } + } + q = append(q, p1.Internal.Imports...) + } + sortedMods := make([]module.Version, 0, len(mdeps)) + for mod := range mdeps { + sortedMods = append(sortedMods, mod) + } + module.Sort(sortedMods) + deps := make([]*debug.Module, len(sortedMods)) + for i, mod := range sortedMods { + deps[i] = mdeps[mod] + } + + pkgPath := p.ImportPath + if p.Internal.CmdlineFiles { + pkgPath = "command-line-arguments" + } + info := &debug.BuildInfo{ + Path: pkgPath, + Main: main, + Deps: deps, + } + appendSetting := func(key, value string) { + value = strings.ReplaceAll(value, "\n", " ") // make value safe + info.Settings = append(info.Settings, debug.BuildSetting{Key: key, Value: value}) + } + + // Add command-line flags relevant to the build. + // This is informational, not an exhaustive list. + // Please keep the list sorted. + if !p.Standard { + if cfg.BuildASan { + appendSetting("-asan", "true") + } + if BuildAsmflags.present { + appendSetting("-asmflags", BuildAsmflags.String()) + } + appendSetting("-compiler", cfg.BuildContext.Compiler) + if BuildGccgoflags.present && cfg.BuildContext.Compiler == "gccgo" { + appendSetting("-gccgoflags", BuildGccgoflags.String()) + } + if BuildGcflags.present && cfg.BuildContext.Compiler == "gc" { + appendSetting("-gcflags", BuildGcflags.String()) + } + if BuildLdflags.present { + appendSetting("-ldflags", BuildLdflags.String()) + } + if cfg.BuildMSan { + appendSetting("-msan", "true") + } + if cfg.BuildRace { + appendSetting("-race", "true") + } + if tags := cfg.BuildContext.BuildTags; len(tags) > 0 { + appendSetting("-tags", strings.Join(tags, ",")) + } + cgo := "0" + if cfg.BuildContext.CgoEnabled { + cgo = "1" + } + appendSetting("CGO_ENABLED", cgo) + if cfg.BuildContext.CgoEnabled { + for _, name := range []string{"CGO_CFLAGS", "CGO_CPPFLAGS", "CGO_CXXFLAGS", "CGO_LDFLAGS"} { + appendSetting(name, cfg.Getenv(name)) + } + } + appendSetting("GOARCH", cfg.BuildContext.GOARCH) + if cfg.GOEXPERIMENT != "" { + appendSetting("GOEXPERIMENT", cfg.GOEXPERIMENT) + } + appendSetting("GOOS", cfg.BuildContext.GOOS) + if key, val := cfg.GetArchEnv(); key != "" && val != "" { + appendSetting(key, val) + } + } + + // Add VCS status if all conditions are true: + // + // - -buildvcs is enabled. + // - p is contained within a main module (there may be multiple main modules + // in a workspace, but local replacements don't count). + // - Both the current directory and p's module's root directory are contained + // in the same local repository. + // - We know the VCS commands needed to get the status. + setVCSError := func(err error) { + setPkgErrorf("error obtaining VCS status: %v\n\tUse -buildvcs=false to disable VCS stamping.", err) + } + + var repoDir string + var vcsCmd *vcs.Cmd + var err error + const allowNesting = true + if cfg.BuildBuildvcs && p.Module != nil && p.Module.Version == "" && !p.Standard { + repoDir, vcsCmd, err = vcs.FromDir(base.Cwd(), "", allowNesting) + if err != nil && !errors.Is(err, os.ErrNotExist) { + setVCSError(err) + return + } + if !str.HasFilePathPrefix(p.Module.Dir, repoDir) && + !str.HasFilePathPrefix(repoDir, p.Module.Dir) { + // The module containing the main package does not overlap with the + // repository containing the working directory. Don't include VCS info. + // If the repo contains the module or vice versa, but they are not + // the same directory, it's likely an error (see below). + repoDir, vcsCmd = "", nil + } + } + if repoDir != "" && vcsCmd.Status != nil { + // Check that the current directory, package, and module are in the same + // repository. vcs.FromDir allows nested Git repositories, but nesting + // is not allowed for other VCS tools. The current directory may be outside + // p.Module.Dir when a workspace is used. + pkgRepoDir, _, err := vcs.FromDir(p.Dir, "", allowNesting) + if err != nil { + setVCSError(err) + return + } + if pkgRepoDir != repoDir { + setVCSError(fmt.Errorf("main package is in repository %q but current directory is in repository %q", pkgRepoDir, repoDir)) + return + } + modRepoDir, _, err := vcs.FromDir(p.Module.Dir, "", allowNesting) + if err != nil { + setVCSError(err) + return + } + if modRepoDir != repoDir { + setVCSError(fmt.Errorf("main module is in repository %q but current directory is in repository %q", modRepoDir, repoDir)) + return + } + + type vcsStatusError struct { + Status vcs.Status + Err error + } + cached := vcsStatusCache.Do(repoDir, func() any { + st, err := vcsCmd.Status(vcsCmd, repoDir) + return vcsStatusError{st, err} + }).(vcsStatusError) + if err := cached.Err; err != nil { + setVCSError(err) + return + } + st := cached.Status + + appendSetting("vcs", vcsCmd.Cmd) + if st.Revision != "" { + appendSetting("vcs.revision", st.Revision) + } + if !st.CommitTime.IsZero() { + stamp := st.CommitTime.UTC().Format(time.RFC3339Nano) + appendSetting("vcs.time", stamp) + } + appendSetting("vcs.modified", strconv.FormatBool(st.Uncommitted)) + } + + text, err := info.MarshalText() + if err != nil { + setPkgErrorf("error formatting build info: %v", err) + return + } + p.Internal.BuildInfo = string(text) +} + // SafeArg reports whether arg is a "safe" command-line argument, // meaning that when it appears in a command-line, it probably // doesn't have some special meaning other than its own name. @@ -2237,6 +2470,10 @@ func LinkerDeps(p *Package) []string { if cfg.BuildMSan { deps = append(deps, "runtime/msan") } + // Using address sanitizer forces an import of runtime/asan. + if cfg.BuildASan { + deps = append(deps, "runtime/asan") + } return deps } @@ -2453,7 +2690,8 @@ func PackagesAndErrors(ctx context.Context, opts PackageOpts, patterns []string) } matches, _ = modload.LoadPackages(ctx, modOpts, patterns...) } else { - matches = search.ImportPaths(patterns) + noModRoots := []string{} + matches = search.ImportPaths(patterns, noModRoots) } var ( @@ -2679,10 +2917,7 @@ func GoFilesPackage(ctx context.Context, opts PackageOpts, gofiles []string) *Pa if fi.IsDir() { base.Fatalf("%s is a directory, should be a Go file", file) } - dir1, _ := filepath.Split(file) - if dir1 == "" { - dir1 = "./" - } + dir1 := filepath.Dir(file) if dir == "" { dir = dir1 } else if dir != dir1 { @@ -2710,7 +2945,9 @@ func GoFilesPackage(ctx context.Context, opts PackageOpts, gofiles []string) *Pa pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true pkg.load(ctx, opts, "command-line-arguments", &stk, nil, bp, err) - pkg.Internal.LocalPrefix = dirToImportPath(dir) + if !cfg.ModulesEnabled { + pkg.Internal.LocalPrefix = dirToImportPath(dir) + } pkg.ImportPath = "command-line-arguments" pkg.Target = "" pkg.Match = gofiles diff --git a/libgo/go/cmd/go/internal/load/test.go b/libgo/go/cmd/go/internal/load/test.go index c8282965669..6122428c9cf 100644 --- a/libgo/go/cmd/go/internal/load/test.go +++ b/libgo/go/cmd/go/internal/load/test.go @@ -555,6 +555,7 @@ func formatTestmain(t *testFuncs) ([]byte, error) { type testFuncs struct { Tests []testFunc Benchmarks []testFunc + FuzzTargets []testFunc Examples []testFunc TestMain *testFunc Package *Package @@ -653,6 +654,13 @@ func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { } t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false}) *doImport, *seen = true, true + case isTest(name, "Fuzz"): + err := checkTestFunc(n, "F") + if err != nil { + return err + } + t.FuzzTargets = append(t.FuzzTargets, testFunc{pkg, name, "", false}) + *doImport, *seen = true, true } } ex := doc.Examples(f) @@ -670,10 +678,16 @@ func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { } func checkTestFunc(fn *ast.FuncDecl, arg string) error { + var why string if !isTestFunc(fn, arg) { - name := fn.Name.String() + why = fmt.Sprintf("must be: func %s(%s *testing.%s)", fn.Name.String(), strings.ToLower(arg), arg) + } + if fn.Type.TypeParams.NumFields() > 0 { + why = "test functions cannot have type parameters" + } + if why != "" { pos := testFileSet.Position(fn.Pos()) - return fmt.Errorf("%s: wrong signature for %s, must be: func %s(%s *testing.%s)", pos, name, name, strings.ToLower(arg), arg) + return fmt.Errorf("%s: wrong signature for %s, %s", pos, fn.Name.String(), why) } return nil } @@ -716,6 +730,12 @@ var benchmarks = []testing.InternalBenchmark{ {{end}} } +var fuzzTargets = []testing.InternalFuzzTarget{ +{{range .FuzzTargets}} + {"{{.Name}}", {{.Package}}.{{.Name}}}, +{{end}} +} + var examples = []testing.InternalExample{ {{range .Examples}} {"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}}, @@ -774,7 +794,7 @@ func main() { CoveredPackages: {{printf "%q" .Covered}}, }) {{end}} - m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) + m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, fuzzTargets, examples) {{with .TestMain}} {{.Package}}.{{.Name}}(m) os.Exit(int(reflect.ValueOf(m).Elem().FieldByName("exitCode").Int())) diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go index e302fe7fccf..31ab5ea4002 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build aix || (solaris && !illumos) -// +build aix solaris,!illumos // This code implements the filelock API using POSIX 'fcntl' locks, which attach // to an (inode, process) pair rather than a file descriptor. To avoid unlocking diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go index c6eac302c57..4aef110bb1a 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !aix && !darwin && !dragonfly && !freebsd && !hurd && !linux && !netbsd && !openbsd && !plan9 && !solaris && !windows -// +build !aix,!darwin,!dragonfly,!freebsd,!hurd,!linux,!netbsd,!openbsd,!plan9,!solaris,!windows package filelock diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go index 908afb6c8cb..54b2c946e0d 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build plan9 -// +build plan9 package filelock diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go index 640d4406f42..7bd7bd28f55 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !js && !plan9 -// +build !js,!plan9 package filelock_test diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go index b22c0bb9918..61427c7c031 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build darwin || dragonfly || freebsd || hurd || illumos || linux || netbsd || openbsd -// +build darwin dragonfly freebsd hurd illumos linux netbsd openbsd package filelock diff --git a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go index dd27ce92bd8..e2ca5383046 100644 --- a/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go +++ b/libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build windows -// +build windows package filelock diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go index e4923f68764..1a677a7fe4a 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !plan9 -// +build !plan9 package lockedfile diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go index 979118b10ae..35669388e05 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build plan9 -// +build plan9 package lockedfile diff --git a/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go b/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go index 3acc6695a74..c9907db46ce 100644 --- a/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go +++ b/libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go @@ -4,7 +4,6 @@ // js does not support inter-process file locking. //go:build !js -// +build !js package lockedfile_test diff --git a/libgo/go/cmd/go/internal/lockedfile/transform_test.go b/libgo/go/cmd/go/internal/lockedfile/transform_test.go index b753346e7da..3c1caa334eb 100644 --- a/libgo/go/cmd/go/internal/lockedfile/transform_test.go +++ b/libgo/go/cmd/go/internal/lockedfile/transform_test.go @@ -4,7 +4,6 @@ // js does not support inter-process file locking. //go:build !js -// +build !js package lockedfile_test diff --git a/libgo/go/cmd/go/internal/modcmd/download.go b/libgo/go/cmd/go/internal/modcmd/download.go index 0e5af852376..6b8a010fd9d 100644 --- a/libgo/go/cmd/go/internal/modcmd/download.go +++ b/libgo/go/cmd/go/internal/modcmd/download.go @@ -16,6 +16,7 @@ import ( "cmd/go/internal/modload" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) var cmdDownload = &base.Command{ @@ -24,8 +25,11 @@ var cmdDownload = &base.Command{ Long: ` Download downloads the named modules, which can be module patterns selecting dependencies of the main module or module queries of the form path@version. -With no arguments, download applies to all dependencies of the main module -(equivalent to 'go mod download all'). + +With no arguments, download applies to the modules needed to build and test +the packages in the main module: the modules explicitly required by the main +module if it is at 'go 1.17' or higher, or all transitively-required modules +if at 'go 1.16' or lower. The go command will automatically download modules as needed during ordinary execution. The "go mod download" command is useful mainly for pre-filling @@ -66,6 +70,7 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") base.AddModCommonFlags(&cmdDownload.Flag) + base.AddWorkfileFlag(&cmdDownload.Flag) } type moduleJSON struct { @@ -81,27 +86,68 @@ type moduleJSON struct { } func runDownload(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + // Check whether modules are enabled and whether we're in a module. modload.ForceUseModules = true - if !modload.HasModRoot() && len(args) == 0 { - base.Fatalf("go mod download: no modules specified (see 'go help mod download')") - } + modload.ExplicitWriteGoMod = true haveExplicitArgs := len(args) > 0 - if !haveExplicitArgs { - args = []string{"all"} - } - if modload.HasModRoot() { - modload.LoadModFile(ctx) // to fill Target - targetAtUpgrade := modload.Target.Path + "@upgrade" - targetAtPatch := modload.Target.Path + "@patch" - for _, arg := range args { - switch arg { - case modload.Target.Path, targetAtUpgrade, targetAtPatch: - os.Stderr.WriteString("go mod download: skipping argument " + arg + " that resolves to the main module\n") + + if modload.HasModRoot() || modload.WorkFilePath() != "" { + modload.LoadModFile(ctx) // to fill MainModules + + if haveExplicitArgs { + for _, mainModule := range modload.MainModules.Versions() { + targetAtUpgrade := mainModule.Path + "@upgrade" + targetAtPatch := mainModule.Path + "@patch" + for _, arg := range args { + switch arg { + case mainModule.Path, targetAtUpgrade, targetAtPatch: + os.Stderr.WriteString("go: skipping download of " + arg + " that resolves to the main module\n") + } + } + } + } else if modload.WorkFilePath() != "" { + // TODO(#44435): Think about what the correct query is to download the + // right set of modules. Also see code review comment at + // https://go-review.googlesource.com/c/go/+/359794/comments/ce946a80_6cf53992. + args = []string{"all"} + } else { + mainModule := modload.MainModules.Versions()[0] + modFile := modload.MainModules.ModFile(mainModule) + if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, modload.ExplicitIndirectVersionV) < 0 { + if len(modFile.Require) > 0 { + args = []string{"all"} + } + } else { + // As of Go 1.17, the go.mod file explicitly requires every module + // that provides any package imported by the main module. + // 'go mod download' is typically run before testing packages in the + // main module, so by default we shouldn't download the others + // (which are presumed irrelevant to the packages in the main module). + // See https://golang.org/issue/44435. + // + // However, we also need to load the full module graph, to ensure that + // we have downloaded enough of the module graph to run 'go list all', + // 'go mod graph', and similar commands. + _ = modload.LoadModGraph(ctx, "") + + for _, m := range modFile.Require { + args = append(args, m.Mod.Path) + } } } } + if len(args) == 0 { + if modload.HasModRoot() { + os.Stderr.WriteString("go: no module dependencies to download\n") + } else { + base.Errorf("go: no modules specified (see 'go help mod download')") + } + base.Exit() + } + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) @@ -140,13 +186,16 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if !haveExplicitArgs { // 'go mod download' is sometimes run without arguments to pre-populate the // module cache. It may fetch modules that aren't needed to build packages - // in the main mdoule. This is usually not intended, so don't save sums for - // downloaded modules (golang.org/issue/45332). - // TODO(golang.org/issue/45551): For now, in ListModules, save sums needed - // to load the build list (same as 1.15 behavior). In the future, report an - // error if go.mod or go.sum need to be updated after loading the build - // list. - modload.DisallowWriteGoMod() + // in the main module. This is usually not intended, so don't save sums for + // downloaded modules (golang.org/issue/45332). We do still fix + // inconsistencies in go.mod though. + // + // TODO(#45551): In the future, report an error if go.mod or go.sum need to + // be updated after loading the build list. This may require setting + // the mode to "mod" or "readonly" depending on haveExplicitArgs. + if err := modload.WriteGoMod(ctx); err != nil { + base.Fatalf("go: %v", err) + } } for _, info := range infos { @@ -183,7 +232,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { for _, m := range mods { b, err := json.MarshalIndent(m, "", "\t") if err != nil { - base.Fatalf("go mod download: %v", err) + base.Fatalf("go: %v", err) } os.Stdout.Write(append(b, '\n')) if m.Error != "" { @@ -193,7 +242,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } else { for _, m := range mods { if m.Error != "" { - base.Errorf("go mod download: %v", m.Error) + base.Errorf("go: %v", m.Error) } } base.ExitIfErrors() @@ -206,13 +255,15 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { // // Don't save sums for 'go mod download' without arguments; see comment above. if haveExplicitArgs { - modload.WriteGoMod(ctx) + if err := modload.WriteGoMod(ctx); err != nil { + base.Errorf("go: %v", err) + } } // If there was an error matching some of the requested packages, emit it now // (after we've written the checksums for the modules that were downloaded // successfully). if infosErr != nil { - base.Errorf("go mod download: %v", infosErr) + base.Errorf("go: %v", infosErr) } } diff --git a/libgo/go/cmd/go/internal/modcmd/edit.go b/libgo/go/cmd/go/internal/modcmd/edit.go index bb3d5210926..e5182a9590a 100644 --- a/libgo/go/cmd/go/internal/modcmd/edit.go +++ b/libgo/go/cmd/go/internal/modcmd/edit.go @@ -171,15 +171,15 @@ func runEdit(ctx context.Context, cmd *base.Command, args []string) { len(edits) > 0 if !anyFlags { - base.Fatalf("go mod edit: no flags specified (see 'go help mod edit').") + base.Fatalf("go: no flags specified (see 'go help mod edit').") } if *editJSON && *editPrint { - base.Fatalf("go mod edit: cannot use both -json and -print") + base.Fatalf("go: cannot use both -json and -print") } if len(args) > 1 { - base.Fatalf("go mod edit: too many arguments") + base.Fatalf("go: too many arguments") } var gomod string if len(args) == 1 { @@ -190,7 +190,7 @@ func runEdit(ctx context.Context, cmd *base.Command, args []string) { if *editModule != "" { if err := module.CheckImportPath(*editModule); err != nil { - base.Fatalf("go mod: invalid -module: %v", err) + base.Fatalf("go: invalid -module: %v", err) } } @@ -264,15 +264,15 @@ func runEdit(ctx context.Context, cmd *base.Command, args []string) { func parsePathVersion(flag, arg string) (path, version string) { i := strings.Index(arg, "@") if i < 0 { - base.Fatalf("go mod: -%s=%s: need path@version", flag, arg) + base.Fatalf("go: -%s=%s: need path@version", flag, arg) } path, version = strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:]) if err := module.CheckImportPath(path); err != nil { - base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err) + base.Fatalf("go: -%s=%s: invalid path: %v", flag, arg, err) } if !allowedVersionArg(version) { - base.Fatalf("go mod: -%s=%s: invalid version %q", flag, arg, version) + base.Fatalf("go: -%s=%s: invalid version %q", flag, arg, version) } return path, version @@ -281,11 +281,11 @@ func parsePathVersion(flag, arg string) (path, version string) { // parsePath parses -flag=arg expecting arg to be path (not path@version). func parsePath(flag, arg string) (path string) { if strings.Contains(arg, "@") { - base.Fatalf("go mod: -%s=%s: need just path, not path@version", flag, arg) + base.Fatalf("go: -%s=%s: need just path, not path@version", flag, arg) } path = arg if err := module.CheckImportPath(path); err != nil { - base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err) + base.Fatalf("go: -%s=%s: invalid path: %v", flag, arg, err) } return path } @@ -350,7 +350,7 @@ func flagRequire(arg string) { path, version := parsePathVersion("require", arg) edits = append(edits, func(f *modfile.File) { if err := f.AddRequire(path, version); err != nil { - base.Fatalf("go mod: -require=%s: %v", arg, err) + base.Fatalf("go: -require=%s: %v", arg, err) } }) } @@ -360,7 +360,7 @@ func flagDropRequire(arg string) { path := parsePath("droprequire", arg) edits = append(edits, func(f *modfile.File) { if err := f.DropRequire(path); err != nil { - base.Fatalf("go mod: -droprequire=%s: %v", arg, err) + base.Fatalf("go: -droprequire=%s: %v", arg, err) } }) } @@ -370,7 +370,7 @@ func flagExclude(arg string) { path, version := parsePathVersion("exclude", arg) edits = append(edits, func(f *modfile.File) { if err := f.AddExclude(path, version); err != nil { - base.Fatalf("go mod: -exclude=%s: %v", arg, err) + base.Fatalf("go: -exclude=%s: %v", arg, err) } }) } @@ -380,7 +380,7 @@ func flagDropExclude(arg string) { path, version := parsePathVersion("dropexclude", arg) edits = append(edits, func(f *modfile.File) { if err := f.DropExclude(path, version); err != nil { - base.Fatalf("go mod: -dropexclude=%s: %v", arg, err) + base.Fatalf("go: -dropexclude=%s: %v", arg, err) } }) } @@ -389,27 +389,27 @@ func flagDropExclude(arg string) { func flagReplace(arg string) { var i int if i = strings.Index(arg, "="); i < 0 { - base.Fatalf("go mod: -replace=%s: need old[@v]=new[@w] (missing =)", arg) + base.Fatalf("go: -replace=%s: need old[@v]=new[@w] (missing =)", arg) } old, new := strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:]) if strings.HasPrefix(new, ">") { - base.Fatalf("go mod: -replace=%s: separator between old and new is =, not =>", arg) + base.Fatalf("go: -replace=%s: separator between old and new is =, not =>", arg) } oldPath, oldVersion, err := parsePathVersionOptional("old", old, false) if err != nil { - base.Fatalf("go mod: -replace=%s: %v", arg, err) + base.Fatalf("go: -replace=%s: %v", arg, err) } newPath, newVersion, err := parsePathVersionOptional("new", new, true) if err != nil { - base.Fatalf("go mod: -replace=%s: %v", arg, err) + base.Fatalf("go: -replace=%s: %v", arg, err) } if newPath == new && !modfile.IsDirectoryPath(new) { - base.Fatalf("go mod: -replace=%s: unversioned new path must be local directory", arg) + base.Fatalf("go: -replace=%s: unversioned new path must be local directory", arg) } edits = append(edits, func(f *modfile.File) { if err := f.AddReplace(oldPath, oldVersion, newPath, newVersion); err != nil { - base.Fatalf("go mod: -replace=%s: %v", arg, err) + base.Fatalf("go: -replace=%s: %v", arg, err) } }) } @@ -418,11 +418,11 @@ func flagReplace(arg string) { func flagDropReplace(arg string) { path, version, err := parsePathVersionOptional("old", arg, true) if err != nil { - base.Fatalf("go mod: -dropreplace=%s: %v", arg, err) + base.Fatalf("go: -dropreplace=%s: %v", arg, err) } edits = append(edits, func(f *modfile.File) { if err := f.DropReplace(path, version); err != nil { - base.Fatalf("go mod: -dropreplace=%s: %v", arg, err) + base.Fatalf("go: -dropreplace=%s: %v", arg, err) } }) } @@ -431,11 +431,11 @@ func flagDropReplace(arg string) { func flagRetract(arg string) { vi, err := parseVersionInterval(arg) if err != nil { - base.Fatalf("go mod: -retract=%s: %v", arg, err) + base.Fatalf("go: -retract=%s: %v", arg, err) } edits = append(edits, func(f *modfile.File) { if err := f.AddRetract(vi, ""); err != nil { - base.Fatalf("go mod: -retract=%s: %v", arg, err) + base.Fatalf("go: -retract=%s: %v", arg, err) } }) } @@ -444,11 +444,11 @@ func flagRetract(arg string) { func flagDropRetract(arg string) { vi, err := parseVersionInterval(arg) if err != nil { - base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + base.Fatalf("go: -dropretract=%s: %v", arg, err) } edits = append(edits, func(f *modfile.File) { if err := f.DropRetract(vi); err != nil { - base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + base.Fatalf("go: -dropretract=%s: %v", arg, err) } }) } diff --git a/libgo/go/cmd/go/internal/modcmd/graph.go b/libgo/go/cmd/go/internal/modcmd/graph.go index ac81f26dade..9b6aa1fb14d 100644 --- a/libgo/go/cmd/go/internal/modcmd/graph.go +++ b/libgo/go/cmd/go/internal/modcmd/graph.go @@ -42,11 +42,14 @@ var ( func init() { cmdGraph.Flag.Var(&graphGo, "go", "") base.AddModCommonFlags(&cmdGraph.Flag) + base.AddWorkfileFlag(&cmdGraph.Flag) } func runGraph(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if len(args) > 0 { - base.Fatalf("go mod graph: graph takes no arguments") + base.Fatalf("go: 'go mod graph' accepts no arguments") } modload.ForceUseModules = true modload.RootMode = modload.NeedRoot diff --git a/libgo/go/cmd/go/internal/modcmd/init.go b/libgo/go/cmd/go/internal/modcmd/init.go index 958c3066ac1..bc4620a2a8d 100644 --- a/libgo/go/cmd/go/internal/modcmd/init.go +++ b/libgo/go/cmd/go/internal/modcmd/init.go @@ -39,7 +39,7 @@ func init() { func runInit(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 1 { - base.Fatalf("go mod init: too many arguments") + base.Fatalf("go: 'go mod init' accepts at most one argument") } var modPath string if len(args) == 1 { diff --git a/libgo/go/cmd/go/internal/modcmd/tidy.go b/libgo/go/cmd/go/internal/modcmd/tidy.go index fe25507e94f..d35476eb539 100644 --- a/libgo/go/cmd/go/internal/modcmd/tidy.go +++ b/libgo/go/cmd/go/internal/modcmd/tidy.go @@ -75,8 +75,8 @@ type goVersionFlag struct { v string } -func (f *goVersionFlag) String() string { return f.v } -func (f *goVersionFlag) Get() interface{} { return f.v } +func (f *goVersionFlag) String() string { return f.v } +func (f *goVersionFlag) Get() any { return f.v } func (f *goVersionFlag) Set(s string) error { if s != "" { @@ -95,7 +95,7 @@ func (f *goVersionFlag) Set(s string) error { func runTidy(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { - base.Fatalf("go mod tidy: no arguments allowed") + base.Fatalf("go: 'go mod tidy' accepts no arguments") } // Tidy aims to make 'go test' reproducible for any package in 'all', so we diff --git a/libgo/go/cmd/go/internal/modcmd/vendor.go b/libgo/go/cmd/go/internal/modcmd/vendor.go index 713d5f9f3fa..ef123700aa3 100644 --- a/libgo/go/cmd/go/internal/modcmd/vendor.go +++ b/libgo/go/cmd/go/internal/modcmd/vendor.go @@ -31,7 +31,7 @@ import ( ) var cmdVendor = &base.Command{ - UsageLine: "go mod vendor [-e] [-v]", + UsageLine: "go mod vendor [-e] [-v] [-o outdir]", Short: "make vendored copy of dependencies", Long: ` Vendor resets the main module's vendor directory to include all packages @@ -44,22 +44,29 @@ modules and packages to standard error. The -e flag causes vendor to attempt to proceed despite errors encountered while loading packages. +The -o flag causes vendor to create the vendor directory at the given +path instead of "vendor". The go command can only use a vendor directory +named "vendor" within the module root directory, so this flag is +primarily useful for other tools. + See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'. `, Run: runVendor, } -var vendorE bool // if true, report errors but proceed anyway +var vendorE bool // if true, report errors but proceed anyway +var vendorO string // if set, overrides the default output directory func init() { cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") cmdVendor.Flag.BoolVar(&vendorE, "e", false, "") + cmdVendor.Flag.StringVar(&vendorO, "o", "", "") base.AddModCommonFlags(&cmdVendor.Flag) } func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { - base.Fatalf("go mod vendor: vendor takes no arguments") + base.Fatalf("go: 'go mod vendor' accepts no arguments") } modload.ForceUseModules = true modload.RootMode = modload.NeedRoot @@ -74,15 +81,23 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") - vdir := filepath.Join(modload.ModRoot(), "vendor") + var vdir string + switch { + case filepath.IsAbs(vendorO): + vdir = vendorO + case vendorO != "": + vdir = filepath.Join(base.Cwd(), vendorO) + default: + vdir = filepath.Join(modload.VendorDir()) + } if err := os.RemoveAll(vdir); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } modpkgs := make(map[module.Version][]string) for _, pkg := range pkgs { m := modload.PackageModule(pkg) - if m.Path == "" || m == modload.Target { + if m.Path == "" || m.Version == "" && modload.MainModules.Contains(m.Path) { continue } modpkgs[m] = append(modpkgs[m], pkg) @@ -128,7 +143,8 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } for _, m := range vendorMods { - line := moduleLine(m, modload.Replacement(m)) + replacement := modload.Replacement(m) + line := moduleLine(m, replacement) io.WriteString(w, line) goVersion := "" @@ -177,11 +193,11 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } if err := os.MkdirAll(vdir, 0777); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } } @@ -242,14 +258,14 @@ func vendorPkg(vdir, pkg string) { if err != nil { if errors.As(err, &noGoError) { return // No source files in this package are built. Skip embeds in ignored files. - } else if !errors.As(err, &multiplePackageError) { // multiplePackgeErrors are okay, but others are not. + } else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not. base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err) } } embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns) embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } for _, embed := range embeds { embedDst := filepath.Join(dst, embed) @@ -260,21 +276,21 @@ func vendorPkg(vdir, pkg string) { // Copy the file as is done by copyDir below. r, err := os.Open(filepath.Join(src, embed)) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } w, err := os.Create(embedDst) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } if _, err := io.Copy(w, r); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } r.Close() if err := w.Close(); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } } } @@ -353,7 +369,7 @@ func matchPotentialSourceFile(dir string, info fs.DirEntry) bool { if strings.HasSuffix(info.Name(), ".go") { f, err := fsys.Open(filepath.Join(dir, info.Name())) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } defer f.Close() @@ -375,10 +391,10 @@ func matchPotentialSourceFile(dir string, info fs.DirEntry) bool { func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) { files, err := os.ReadDir(src) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } if err := os.MkdirAll(dst, 0777); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } for _, file := range files { if file.IsDir() || !file.Type().IsRegular() || !match(src, file) { @@ -387,20 +403,20 @@ func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, cop copiedFiles[file.Name()] = true r, err := os.Open(filepath.Join(src, file.Name())) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } dstPath := filepath.Join(dst, file.Name()) copiedFiles[dstPath] = true w, err := os.Create(dstPath) if err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } if _, err := io.Copy(w, r); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } r.Close() if err := w.Close(); err != nil { - base.Fatalf("go mod vendor: %v", err) + base.Fatalf("go: %v", err) } } } diff --git a/libgo/go/cmd/go/internal/modcmd/verify.go b/libgo/go/cmd/go/internal/modcmd/verify.go index 5a6eca32cfb..3f0c005d5d9 100644 --- a/libgo/go/cmd/go/internal/modcmd/verify.go +++ b/libgo/go/cmd/go/internal/modcmd/verify.go @@ -39,12 +39,15 @@ See https://golang.org/ref/mod#go-mod-verify for more about 'go mod verify'. func init() { base.AddModCommonFlags(&cmdVerify.Flag) + base.AddWorkfileFlag(&cmdVerify.Flag) } func runVerify(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if len(args) != 0 { // NOTE(rsc): Could take a module pattern. - base.Fatalf("go mod verify: verify takes no arguments") + base.Fatalf("go: verify takes no arguments") } modload.ForceUseModules = true modload.RootMode = modload.NeedRoot diff --git a/libgo/go/cmd/go/internal/modcmd/why.go b/libgo/go/cmd/go/internal/modcmd/why.go index 3b14b27c8c7..d8355cca957 100644 --- a/libgo/go/cmd/go/internal/modcmd/why.go +++ b/libgo/go/cmd/go/internal/modcmd/why.go @@ -12,8 +12,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/imports" "cmd/go/internal/modload" - - "golang.org/x/mod/module" ) var cmdWhy = &base.Command{ @@ -61,11 +59,14 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle base.AddModCommonFlags(&cmdWhy.Flag) + base.AddWorkfileFlag(&cmdWhy.Flag) } func runWhy(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() modload.ForceUseModules = true modload.RootMode = modload.NeedRoot + modload.ExplicitWriteGoMod = true // don't write go.mod in ListModules loadOpts := modload.PackageOpts{ Tags: imports.AnyTags(), @@ -78,28 +79,28 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { if *whyM { for _, arg := range args { if strings.Contains(arg, "@") { - base.Fatalf("go mod why: module query not allowed") + base.Fatalf("go: %s: 'go mod why' requires a module path, not a version query", arg) } } mods, err := modload.ListModules(ctx, args, 0) if err != nil { - base.Fatalf("go mod why: %v", err) + base.Fatalf("go: %v", err) } - byModule := make(map[module.Version][]string) + byModule := make(map[string][]string) _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") for _, path := range pkgs { m := modload.PackageModule(path) if m.Path != "" { - byModule[m] = append(byModule[m], path) + byModule[m.Path] = append(byModule[m.Path], path) } } sep := "" for _, m := range mods { best := "" bestDepth := 1000000000 - for _, path := range byModule[module.Version{Path: m.Path, Version: m.Version}] { + for _, path := range byModule[m.Path] { d := modload.WhyDepth(path) if d > 0 && d < bestDepth { best = path diff --git a/libgo/go/cmd/go/internal/modfetch/bootstrap.go b/libgo/go/cmd/go/internal/modfetch/bootstrap.go index ed694581a7c..e23669fb00c 100644 --- a/libgo/go/cmd/go/internal/modfetch/bootstrap.go +++ b/libgo/go/cmd/go/internal/modfetch/bootstrap.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build cmd_go_bootstrap -// +build cmd_go_bootstrap package modfetch diff --git a/libgo/go/cmd/go/internal/modfetch/cache.go b/libgo/go/cmd/go/internal/modfetch/cache.go index b01b4674131..b0dae1cb3d3 100644 --- a/libgo/go/cmd/go/internal/modfetch/cache.go +++ b/libgo/go/cmd/go/internal/modfetch/cache.go @@ -204,7 +204,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) { list []string err error } - c := r.cache.Do("versions:"+prefix, func() interface{} { + c := r.cache.Do("versions:"+prefix, func() any { list, err := r.repo().Versions(prefix) return cached{list, err} }).(cached) @@ -221,7 +221,7 @@ type cachedInfo struct { } func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { - c := r.cache.Do("stat:"+rev, func() interface{} { + c := r.cache.Do("stat:"+rev, func() any { file, info, err := readDiskStat(r.path, rev) if err == nil { return cachedInfo{info, nil} @@ -233,7 +233,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { // then save the information under the proper version, for future use. if info.Version != rev { file, _ = CachePath(module.Version{Path: r.path, Version: info.Version}, "info") - r.cache.Do("stat:"+info.Version, func() interface{} { + r.cache.Do("stat:"+info.Version, func() any { return cachedInfo{info, err} }) } @@ -253,12 +253,12 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { } func (r *cachingRepo) Latest() (*RevInfo, error) { - c := r.cache.Do("latest:", func() interface{} { + c := r.cache.Do("latest:", func() any { info, err := r.repo().Latest() // Save info for likely future Stat call. if err == nil { - r.cache.Do("stat:"+info.Version, func() interface{} { + r.cache.Do("stat:"+info.Version, func() any { return cachedInfo{info, err} }) if file, _, err := readDiskStat(r.path, info.Version); err != nil { @@ -281,7 +281,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) { text []byte err error } - c := r.cache.Do("gomod:"+version, func() interface{} { + c := r.cache.Do("gomod:"+version, func() any { file, text, err := readDiskGoMod(r.path, version) if err == nil { // Note: readDiskGoMod already called checkGoMod. @@ -642,7 +642,7 @@ func rewriteVersionList(dir string) (err error) { // Lock listfile when writing to it to try to avoid corruption to the file. // Under rare circumstances, for instance, if the system loses power in the // middle of a write it is possible for corrupt data to be written. This is - // not a problem for the go command itself, but may be an issue if the the + // not a problem for the go command itself, but may be an issue if the // cache is being served by a GOPROXY HTTP server. This will be corrected // the next time a new version of the module is fetched and the file is rewritten. // TODO(matloob): golang.org/issue/43313 covers adding a go mod verify @@ -720,7 +720,7 @@ func checkCacheDir() error { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. - return fmt.Errorf("internal error: cfg.GOMODCACHE not set") + return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set") } if !filepath.IsAbs(cfg.GOMODCACHE) { return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE) diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go b/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go index 378fbae34f9..4a0e2241e50 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/codehost.go @@ -55,21 +55,6 @@ type Repo interface { // os.IsNotExist(err) returns true. ReadFile(rev, file string, maxSize int64) (data []byte, err error) - // ReadFileRevs reads a single file at multiple versions. - // It should refuse to read more than maxSize bytes. - // The result is a map from each requested rev strings - // to the associated FileRev. The map must have a non-nil - // entry for every requested rev (unless ReadFileRevs returned an error). - // A file simply being missing or even corrupted in revs[i] - // should be reported only in files[revs[i]].Err, not in the error result - // from ReadFileRevs. - // The overall call should return an error (and no map) only - // in the case of a problem with obtaining the data, such as - // a network failure. - // Implementations may assume that revs only contain tags, - // not direct commit hashes. - ReadFileRevs(revs []string, file string, maxSize int64) (files map[string]*FileRev, err error) - // ReadZip downloads a zip file for the subdir subdirectory // of the given revision to a new file in a given temporary directory. // It should refuse to read more than maxSize bytes. @@ -243,7 +228,7 @@ var dirLock sync.Map // It returns the standard output and, for a non-zero exit, // a *RunError indicating the command, exit status, and standard error. // Standard error is unavailable for commands that exit successfully. -func Run(dir string, cmdline ...interface{}) ([]byte, error) { +func Run(dir string, cmdline ...any) ([]byte, error) { return RunWithStdin(dir, nil, cmdline...) } @@ -251,7 +236,7 @@ func Run(dir string, cmdline ...interface{}) ([]byte, error) { // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html. var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`) -func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, error) { +func RunWithStdin(dir string, stdin io.Reader, cmdline ...any) ([]byte, error) { if dir != "" { muIface, ok := dirLock.Load(dir) if !ok { diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/git.go b/libgo/go/cmd/go/internal/modfetch/codehost/git.go index 4d4964edf44..34f453c855e 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/git.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/git.go @@ -56,7 +56,7 @@ func newGitRepoCached(remote string, localOK bool) (Repo, error) { err error } - c := gitRepoCache.Do(key{remote, localOK}, func() interface{} { + c := gitRepoCache.Do(key{remote, localOK}, func() any { repo, err := newGitRepo(remote, localOK) return cached{repo, err} }).(cached) @@ -170,59 +170,63 @@ func (r *gitRepo) loadLocalTags() { } // loadRefs loads heads and tags references from the remote into the map r.refs. -// Should only be called as r.refsOnce.Do(r.loadRefs). -func (r *gitRepo) loadRefs() { - // The git protocol sends all known refs and ls-remote filters them on the client side, - // so we might as well record both heads and tags in one shot. - // Most of the time we only care about tags but sometimes we care about heads too. - out, gitErr := Run(r.dir, "git", "ls-remote", "-q", r.remote) - if gitErr != nil { - if rerr, ok := gitErr.(*RunError); ok { - if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) { - rerr.HelpText = "Confirm the import path was entered correctly.\nIf this is a private repository, see https://golang.org/doc/faq#git_https for additional information." +// The result is cached in memory. +func (r *gitRepo) loadRefs() (map[string]string, error) { + r.refsOnce.Do(func() { + // The git protocol sends all known refs and ls-remote filters them on the client side, + // so we might as well record both heads and tags in one shot. + // Most of the time we only care about tags but sometimes we care about heads too. + out, gitErr := Run(r.dir, "git", "ls-remote", "-q", r.remote) + if gitErr != nil { + if rerr, ok := gitErr.(*RunError); ok { + if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) { + rerr.HelpText = "Confirm the import path was entered correctly.\nIf this is a private repository, see https://golang.org/doc/faq#git_https for additional information." + } } - } - // If the remote URL doesn't exist at all, ideally we should treat the whole - // repository as nonexistent by wrapping the error in a notExistError. - // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL - // ourselves and see what code it serves. - if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { - if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { - gitErr = notExistError{gitErr} + // If the remote URL doesn't exist at all, ideally we should treat the whole + // repository as nonexistent by wrapping the error in a notExistError. + // For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL + // ourselves and see what code it serves. + if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") { + if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) { + gitErr = notExistError{gitErr} + } } - } - r.refsErr = gitErr - return - } - - r.refs = make(map[string]string) - for _, line := range strings.Split(string(out), "\n") { - f := strings.Fields(line) - if len(f) != 2 { - continue + r.refsErr = gitErr + return } - if f[1] == "HEAD" || strings.HasPrefix(f[1], "refs/heads/") || strings.HasPrefix(f[1], "refs/tags/") { - r.refs[f[1]] = f[0] + + refs := make(map[string]string) + for _, line := range strings.Split(string(out), "\n") { + f := strings.Fields(line) + if len(f) != 2 { + continue + } + if f[1] == "HEAD" || strings.HasPrefix(f[1], "refs/heads/") || strings.HasPrefix(f[1], "refs/tags/") { + refs[f[1]] = f[0] + } } - } - for ref, hash := range r.refs { - if strings.HasSuffix(ref, "^{}") { // record unwrapped annotated tag as value of tag - r.refs[strings.TrimSuffix(ref, "^{}")] = hash - delete(r.refs, ref) + for ref, hash := range refs { + if strings.HasSuffix(ref, "^{}") { // record unwrapped annotated tag as value of tag + refs[strings.TrimSuffix(ref, "^{}")] = hash + delete(refs, ref) + } } - } + r.refs = refs + }) + return r.refs, r.refsErr } func (r *gitRepo) Tags(prefix string) ([]string, error) { - r.refsOnce.Do(r.loadRefs) - if r.refsErr != nil { - return nil, r.refsErr + refs, err := r.loadRefs() + if err != nil { + return nil, err } tags := []string{} - for ref := range r.refs { + for ref := range refs { if !strings.HasPrefix(ref, "refs/tags/") { continue } @@ -237,14 +241,14 @@ func (r *gitRepo) Tags(prefix string) ([]string, error) { } func (r *gitRepo) Latest() (*RevInfo, error) { - r.refsOnce.Do(r.loadRefs) - if r.refsErr != nil { - return nil, r.refsErr + refs, err := r.loadRefs() + if err != nil { + return nil, err } - if r.refs["HEAD"] == "" { + if refs["HEAD"] == "" { return nil, ErrNoCommits } - return r.Stat(r.refs["HEAD"]) + return r.Stat(refs["HEAD"]) } // findRef finds some ref name for the given hash, @@ -252,8 +256,11 @@ func (r *gitRepo) Latest() (*RevInfo, error) { // There may be multiple ref names for a given hash, // in which case this returns some name - it doesn't matter which. func (r *gitRepo) findRef(hash string) (ref string, ok bool) { - r.refsOnce.Do(r.loadRefs) - for ref, h := range r.refs { + refs, err := r.loadRefs() + if err != nil { + return "", false + } + for ref, h := range refs { if h == hash { return ref, true } @@ -295,29 +302,32 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) { // Maybe rev is the name of a tag or branch on the remote server. // Or maybe it's the prefix of a hash of a named ref. // Try to resolve to both a ref (git name) and full (40-hex-digit) commit hash. - r.refsOnce.Do(r.loadRefs) + refs, err := r.loadRefs() + if err != nil { + return nil, err + } // loadRefs may return an error if git fails, for example segfaults, or // could not load a private repo, but defer checking to the else block // below, in case we already have the rev in question in the local cache. var ref, hash string - if r.refs["refs/tags/"+rev] != "" { + if refs["refs/tags/"+rev] != "" { ref = "refs/tags/" + rev - hash = r.refs[ref] + hash = refs[ref] // Keep rev as is: tags are assumed not to change meaning. - } else if r.refs["refs/heads/"+rev] != "" { + } else if refs["refs/heads/"+rev] != "" { ref = "refs/heads/" + rev - hash = r.refs[ref] + hash = refs[ref] rev = hash // Replace rev, because meaning of refs/heads/foo can change. - } else if rev == "HEAD" && r.refs["HEAD"] != "" { + } else if rev == "HEAD" && refs["HEAD"] != "" { ref = "HEAD" - hash = r.refs[ref] + hash = refs[ref] rev = hash // Replace rev, because meaning of HEAD can change. } else if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) { // At the least, we have a hash prefix we can look up after the fetch below. // Maybe we can map it to a full hash using the known refs. prefix := rev // Check whether rev is prefix of known ref hash. - for k, h := range r.refs { + for k, h := range refs { if strings.HasPrefix(h, prefix) { if hash != "" && hash != h { // Hash is an ambiguous hash prefix. @@ -335,9 +345,6 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) { hash = rev } } else { - if r.refsErr != nil { - return nil, r.refsErr - } return nil, &UnknownRevisionError{Rev: rev} } @@ -496,7 +503,7 @@ func (r *gitRepo) Stat(rev string) (*RevInfo, error) { info *RevInfo err error } - c := r.statCache.Do(rev, func() interface{} { + c := r.statCache.Do(rev, func() any { info, err := r.stat(rev) return cached{info, err} }).(cached) @@ -516,140 +523,6 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { return out, nil } -func (r *gitRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) { - // Create space to hold results. - files := make(map[string]*FileRev) - for _, rev := range revs { - f := &FileRev{Rev: rev} - files[rev] = f - } - - // Collect locally-known revs. - need, err := r.readFileRevs(revs, file, files) - if err != nil { - return nil, err - } - if len(need) == 0 { - return files, nil - } - - // Build list of known remote refs that might help. - var redo []string - r.refsOnce.Do(r.loadRefs) - if r.refsErr != nil { - return nil, r.refsErr - } - for _, tag := range need { - if r.refs["refs/tags/"+tag] != "" { - redo = append(redo, tag) - } - } - if len(redo) == 0 { - return files, nil - } - - // Protect r.fetchLevel and the "fetch more and more" sequence. - // See stat method above. - unlock, err := r.mu.Lock() - if err != nil { - return nil, err - } - defer unlock() - - if err := r.fetchRefsLocked(); err != nil { - return nil, err - } - - if _, err := r.readFileRevs(redo, file, files); err != nil { - return nil, err - } - - return files, nil -} - -func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*FileRev) (missing []string, err error) { - var stdin bytes.Buffer - for _, tag := range tags { - fmt.Fprintf(&stdin, "refs/tags/%s\n", tag) - fmt.Fprintf(&stdin, "refs/tags/%s:%s\n", tag, file) - } - - data, err := RunWithStdin(r.dir, &stdin, "git", "cat-file", "--batch") - if err != nil { - return nil, err - } - - next := func() (typ string, body []byte, ok bool) { - var line string - i := bytes.IndexByte(data, '\n') - if i < 0 { - return "", nil, false - } - line, data = string(bytes.TrimSpace(data[:i])), data[i+1:] - if strings.HasSuffix(line, " missing") { - return "missing", nil, true - } - f := strings.Fields(line) - if len(f) != 3 { - return "", nil, false - } - n, err := strconv.Atoi(f[2]) - if err != nil || n > len(data) { - return "", nil, false - } - body, data = data[:n], data[n:] - if len(data) > 0 && data[0] == '\r' { - data = data[1:] - } - if len(data) > 0 && data[0] == '\n' { - data = data[1:] - } - return f[1], body, true - } - - badGit := func() ([]string, error) { - return nil, fmt.Errorf("malformed output from git cat-file --batch") - } - - for _, tag := range tags { - commitType, _, ok := next() - if !ok { - return badGit() - } - fileType, fileData, ok := next() - if !ok { - return badGit() - } - f := fileMap[tag] - f.Data = nil - f.Err = nil - switch commitType { - default: - f.Err = fmt.Errorf("unexpected non-commit type %q for rev %s", commitType, tag) - - case "missing": - // Note: f.Err must not satisfy os.IsNotExist. That's reserved for the file not existing in a valid commit. - f.Err = fmt.Errorf("no such rev %s", tag) - missing = append(missing, tag) - - case "tag", "commit": - switch fileType { - default: - f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)} - case "missing": - f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist} - case "blob": - f.Data = fileData - } - } - } - if len(bytes.TrimSpace(data)) != 0 { - return badGit() - } - - return missing, nil -} - func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { info, err := r.Stat(rev) if err != nil { diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/shell.go b/libgo/go/cmd/go/internal/modfetch/codehost/shell.go index 0e9f3819667..eaa01950b95 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/shell.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/shell.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ignore -// +build ignore // Interactive debugging shell for codehost.Repo implementations. diff --git a/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go b/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go index c2cca084e30..de62265efc5 100644 --- a/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go +++ b/libgo/go/cmd/go/internal/modfetch/codehost/vcs.go @@ -38,7 +38,7 @@ type VCSError struct { func (e *VCSError) Error() string { return e.Err.Error() } -func vcsErrorf(format string, a ...interface{}) error { +func vcsErrorf(format string, a ...any) error { return &VCSError{Err: fmt.Errorf(format, a...)} } @@ -51,7 +51,7 @@ func NewRepo(vcs, remote string) (Repo, error) { repo Repo err error } - c := vcsRepoCache.Do(key{vcs, remote}, func() interface{} { + c := vcsRepoCache.Do(key{vcs, remote}, func() any { repo, err := newVCSRepo(vcs, remote) if err != nil { err = &VCSError{err} @@ -382,19 +382,6 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { return out, nil } -func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) { - // We don't technically need to lock here since we're returning an error - // uncondititonally, but doing so anyway will help to avoid baking in - // lock-inversion bugs. - unlock, err := r.mu.Lock() - if err != nil { - return nil, err - } - defer unlock() - - return nil, vcsErrorf("ReadFileRevs not implemented") -} - func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { // We don't technically need to lock here since we're returning an error // uncondititonally, but doing so anyway will help to avoid baking in diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo.go b/libgo/go/cmd/go/internal/modfetch/coderepo.go index dfef9f73c27..79da010809b 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo.go @@ -321,7 +321,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return ok } - invalidf := func(format string, args ...interface{}) error { + invalidf := func(format string, args ...any) error { return &module.ModuleError{ Path: r.modPath, Err: &module.InvalidVersionError{ @@ -567,11 +567,11 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string) if rev != info.Short { switch { case strings.HasPrefix(rev, info.Short): - return fmt.Errorf("revision is longer than canonical (%s)", info.Short) + return fmt.Errorf("revision is longer than canonical (expected %s)", info.Short) case strings.HasPrefix(info.Short, rev): - return fmt.Errorf("revision is shorter than canonical (%s)", info.Short) + return fmt.Errorf("revision is shorter than canonical (expected %s)", info.Short) default: - return fmt.Errorf("does not match short name of revision (%s)", info.Short) + return fmt.Errorf("does not match short name of revision (expected %s)", info.Short) } } @@ -1066,7 +1066,7 @@ func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) } func (fi dataFileInfo) Mode() fs.FileMode { return 0644 } func (fi dataFileInfo) ModTime() time.Time { return time.Time{} } func (fi dataFileInfo) IsDir() bool { return false } -func (fi dataFileInfo) Sys() interface{} { return nil } +func (fi dataFileInfo) Sys() any { return nil } // hasPathPrefix reports whether the path s begins with the // elements in prefix. diff --git a/libgo/go/cmd/go/internal/modfetch/fetch.go b/libgo/go/cmd/go/internal/modfetch/fetch.go index d3d30d970b3..f5423b48ad3 100644 --- a/libgo/go/cmd/go/internal/modfetch/fetch.go +++ b/libgo/go/cmd/go/internal/modfetch/fetch.go @@ -48,7 +48,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { dir string err error } - c := downloadCache.Do(mod, func() interface{} { + c := downloadCache.Do(mod, func() any { dir, err := download(ctx, mod) if err != nil { return cached{"", err} @@ -165,7 +165,7 @@ func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err e zipfile string err error } - c := downloadZipCache.Do(mod, func() interface{} { + c := downloadZipCache.Do(mod, func() any { zipfile, err := CachePath(mod, "zip") if err != nil { return cached{"", err} @@ -384,7 +384,8 @@ func RemoveAll(dir string) error { return robustio.RemoveAll(dir) } -var GoSumFile string // path to go.sum; set by package modload +var GoSumFile string // path to go.sum; set by package modload +var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload type modSum struct { mod module.Version @@ -393,16 +394,39 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file - status map[modSum]modSumStatus // state of sums in m - overwrite bool // if true, overwrite go.sum without incorporating its contents - enabled bool // whether to use go.sum at all + m map[module.Version][]string // content of go.sum file + w map[string]map[module.Version][]string // sum file in workspace -> content of that sum file + status map[modSum]modSumStatus // state of sums in m + overwrite bool // if true, overwrite go.sum without incorporating its contents + enabled bool // whether to use go.sum at all } type modSumStatus struct { used, dirty bool } +// Reset resets globals in the modfetch package, so previous loads don't affect +// contents of go.sum files +func Reset() { + GoSumFile = "" + WorkspaceGoSumFiles = nil + + // Uses of lookupCache and downloadCache both can call checkModSum, + // which in turn sets the used bit on goSum.status for modules. + // Reset them so used can be computed properly. + lookupCache = par.Cache{} + downloadCache = par.Cache{} + + // Clear all fields on goSum. It will be initialized later + goSum.mu.Lock() + goSum.m = nil + goSum.w = nil + goSum.status = nil + goSum.overwrite = false + goSum.enabled = false + goSum.mu.Unlock() +} + // initGoSum initializes the go.sum data. // The boolean it returns reports whether the // use of go.sum is now enabled. @@ -417,23 +441,38 @@ func initGoSum() (bool, error) { goSum.m = make(map[module.Version][]string) goSum.status = make(map[modSum]modSumStatus) + goSum.w = make(map[string]map[module.Version][]string) + + for _, f := range WorkspaceGoSumFiles { + goSum.w[f] = make(map[module.Version][]string) + _, err := readGoSumFile(goSum.w[f], f) + if err != nil { + return false, err + } + } + + enabled, err := readGoSumFile(goSum.m, GoSumFile) + goSum.enabled = enabled + return enabled, err +} + +func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) { var ( data []byte err error ) - if actualSumFile, ok := fsys.OverlayPath(GoSumFile); ok { + if actualSumFile, ok := fsys.OverlayPath(file); ok { // Don't lock go.sum if it's part of the overlay. // On Plan 9, locking requires chmod, and we don't want to modify any file // in the overlay. See #44700. data, err = os.ReadFile(actualSumFile) } else { - data, err = lockedfile.Read(GoSumFile) + data, err = lockedfile.Read(file) } if err != nil && !os.IsNotExist(err) { return false, err } - goSum.enabled = true - readGoSum(goSum.m, GoSumFile, data) + readGoSum(dst, file, data) return true, nil } @@ -485,6 +524,16 @@ func HaveSum(mod module.Version) bool { if err != nil || !inited { return false } + for _, goSums := range goSum.w { + for _, h := range goSums[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + } for _, h := range goSum.m[mod] { if !strings.HasPrefix(h, "h1:") { continue @@ -602,15 +651,32 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { + sumFileName := "go.sum" + if strings.HasSuffix(GoSumFile, "go.work.sum") { + sumFileName = "go.work.sum" + } for _, vh := range goSum.m[mod] { if h == vh { return true } if strings.HasPrefix(vh, "h1:") { - base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v"+goSumMismatch, mod.Path, mod.Version, h, vh) + base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh) + } + } + // Also check workspace sums. + foundMatch := false + // Check sums from all files in case there are conflicts between + // the files. + for goSumFile, goSums := range goSum.w { + for _, vh := range goSums[mod] { + if h == vh { + foundMatch = true + } else if strings.HasPrefix(vh, "h1:") { + base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh) + } } } - return false + return foundMatch } // addModSumLocked adds the pair mod,h to go.sum. @@ -693,19 +759,21 @@ func isValidSum(data []byte) bool { return true } +var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly") + // WriteGoSum writes the go.sum file if it needs to be updated. // // keep is used to check whether a newly added sum should be saved in go.sum. // It should have entries for both module content sums and go.mod sums // (version ends with "/go.mod"). Existing sums will be preserved unless they // have been marked for deletion with TrimGoSum. -func WriteGoSum(keep map[module.Version]bool) { +func WriteGoSum(keep map[module.Version]bool, readonly bool) error { goSum.mu.Lock() defer goSum.mu.Unlock() // If we haven't read the go.sum file yet, don't bother writing it. if !goSum.enabled { - return + return nil } // Check whether we need to add sums for which keep[m] is true or remove @@ -723,10 +791,10 @@ Outer: } } if !dirty { - return + return nil } - if cfg.BuildMod == "readonly" { - base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly") + if readonly { + return ErrGoSumDirty } if _, ok := fsys.OverlayPath(GoSumFile); ok { base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay") @@ -747,7 +815,7 @@ Outer: goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) for ms, st := range goSum.status { - if st.used { + if st.used && !sumInWorkspaceModulesLocked(ms.mod) { addModSumLocked(ms.mod, ms.sum) } } @@ -765,7 +833,7 @@ Outer: sort.Strings(list) for _, h := range list { st := goSum.status[modSum{m, h}] - if !st.dirty || (st.used && keep[m]) { + if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) { fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) } } @@ -774,11 +842,21 @@ Outer: }) if err != nil { - base.Fatalf("go: updating go.sum: %v", err) + return fmt.Errorf("updating go.sum: %w", err) } goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false + return nil +} + +func sumInWorkspaceModulesLocked(m module.Version) bool { + for _, goSums := range goSum.w { + if _, ok := goSums[m]; ok { + return true + } + } + return false } // TrimGoSum trims go.sum to contain only the modules needed for reproducible diff --git a/libgo/go/cmd/go/internal/modfetch/repo.go b/libgo/go/cmd/go/internal/modfetch/repo.go index 0bffa55af6f..1b42ecb6edb 100644 --- a/libgo/go/cmd/go/internal/modfetch/repo.go +++ b/libgo/go/cmd/go/internal/modfetch/repo.go @@ -196,7 +196,7 @@ func Lookup(proxy, path string) Repo { type cached struct { r Repo } - c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} { + c := lookupCache.Do(lookupCacheKey{proxy, path}, func() any { r := newCachingRepo(path, func() (Repo, error) { r, err := lookup(proxy, path) if err == nil && traceRepo { @@ -308,7 +308,7 @@ func newLoggingRepo(r Repo) *loggingRepo { // defer logCall("hello %s", arg)() // // Note the final (). -func logCall(format string, args ...interface{}) func() { +func logCall(format string, args ...any) func() { start := time.Now() fmt.Fprintf(os.Stderr, "+++ %s\n", fmt.Sprintf(format, args...)) return func() { @@ -371,7 +371,7 @@ type notExistError struct { err error } -func notExistErrorf(format string, args ...interface{}) error { +func notExistErrorf(format string, args ...any) error { return notExistError{fmt.Errorf(format, args...)} } diff --git a/libgo/go/cmd/go/internal/modfetch/sumdb.go b/libgo/go/cmd/go/internal/modfetch/sumdb.go index f233cba6df1..492b03bd84a 100644 --- a/libgo/go/cmd/go/internal/modfetch/sumdb.go +++ b/libgo/go/cmd/go/internal/modfetch/sumdb.go @@ -5,7 +5,6 @@ // Go checksum database lookup //go:build !cmd_go_bootstrap -// +build !cmd_go_bootstrap package modfetch @@ -201,7 +200,8 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) { } if cfg.SumdbDir == "" { - return nil, errors.New("could not locate sumdb file: missing $GOPATH") + return nil, fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s", + cfg.GoPathError) } targ := filepath.Join(cfg.SumdbDir, file) data, err = lockedfile.Read(targ) @@ -220,7 +220,8 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error { return fmt.Errorf("cannot write key") } if cfg.SumdbDir == "" { - return errors.New("could not locate sumdb file: missing $GOPATH") + return fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s", + cfg.GoPathError) } targ := filepath.Join(cfg.SumdbDir, file) os.MkdirAll(filepath.Dir(targ), 0777) diff --git a/libgo/go/cmd/go/internal/modget/get.go b/libgo/go/cmd/go/internal/modget/get.go index 9672e5598e0..3d8463e892c 100644 --- a/libgo/go/cmd/go/internal/modget/get.go +++ b/libgo/go/cmd/go/internal/modget/get.go @@ -37,7 +37,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/imports" - "cmd/go/internal/load" "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/par" @@ -50,14 +49,14 @@ import ( ) var CmdGet = &base.Command{ - // Note: -d -u are listed explicitly because they are the most common get flags. + // Note: flags below are listed explicitly because they're the most common. // Do not send CLs removing them because they're covered by [get flags]. - UsageLine: "go get [-d] [-t] [-u] [-v] [build flags] [packages]", + UsageLine: "go get [-t] [-u] [-v] [build flags] [packages]", Short: "add dependencies to current module and install them", Long: ` Get resolves its command-line arguments to packages at specific module versions, -updates go.mod to require those versions, downloads source code into the -module cache, then builds and installs the named packages. +updates go.mod to require those versions, and downloads source code into the +module cache. To add a dependency for a package or upgrade it to its latest version: @@ -73,17 +72,18 @@ To remove a dependency on a module and downgrade modules that require it: See https://golang.org/ref/mod#go-get for details. -The 'go install' command may be used to build and install packages. When a -version is specified, 'go install' runs in module-aware mode and ignores -the go.mod file in the current directory. For example: +In earlier versions of Go, 'go get' was used to build and install packages. +Now, 'go get' is dedicated to adjusting dependencies in go.mod. 'go install' +may be used to build and install commands instead. When a version is specified, +'go install' runs in module-aware mode and ignores the go.mod file in the +current directory. For example: go install example.com/pkg@v1.2.3 go install example.com/pkg@latest See 'go help install' or https://golang.org/ref/mod#go-install for details. -In addition to build flags (listed in 'go help build') 'go get' accepts the -following flags. +'go get' accepts the following flags. The -t flag instructs get to consider modules needed to build tests of packages specified on the command line. @@ -98,15 +98,9 @@ but changes the default to select patch releases. When the -t and -u flags are used together, get will update test dependencies as well. -The -d flag instructs get not to build or install packages. get will only -update go.mod and download source code needed to build packages. - -Building and installing packages with get is deprecated. In a future release, -the -d flag will be enabled by default, and 'go get' will be only be used to -adjust dependencies of the current module. To install a package using -dependencies from the current module, use 'go install'. To install a package -ignoring the current module, use 'go install' with an @version suffix like -"@latest" after each argument. +The -x flag prints commands as they are executed. This is useful for +debugging version control commands when a module is downloaded directly +from a repository. For more about modules, see https://golang.org/ref/mod. @@ -218,7 +212,7 @@ variable for future go command invocations. } var ( - getD = CmdGet.Flag.Bool("d", false, "") + getD = CmdGet.Flag.Bool("d", true, "") getF = CmdGet.Flag.Bool("f", false, "") getFix = CmdGet.Flag.Bool("fix", false, "") getM = CmdGet.Flag.Bool("m", false, "") @@ -263,30 +257,50 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { case "", "upgrade", "patch": // ok default: - base.Fatalf("go get: unknown upgrade flag -u=%s", getU.rawVersion) + base.Fatalf("go: unknown upgrade flag -u=%s", getU.rawVersion) + } + // TODO(#43684): in the future (Go 1.20), warn that -d is a no-op. + if !*getD { + base.Fatalf("go: -d flag may not be disabled") } if *getF { - fmt.Fprintf(os.Stderr, "go get: -f flag is a no-op when using modules\n") + fmt.Fprintf(os.Stderr, "go: -f flag is a no-op when using modules\n") } if *getFix { - fmt.Fprintf(os.Stderr, "go get: -fix flag is a no-op when using modules\n") + fmt.Fprintf(os.Stderr, "go: -fix flag is a no-op when using modules\n") } if *getM { - base.Fatalf("go get: -m flag is no longer supported; consider -d to skip building packages") + base.Fatalf("go: -m flag is no longer supported") } if *getInsecure { - base.Fatalf("go get: -insecure flag is no longer supported; use GOINSECURE instead") + base.Fatalf("go: -insecure flag is no longer supported; use GOINSECURE instead") } + modload.ForceUseModules = true + // Do not allow any updating of go.mod until we've applied // all the requested changes and checked that the result matches // what was requested. - modload.DisallowWriteGoMod() + modload.ExplicitWriteGoMod = true // Allow looking up modules for import paths when outside of a module. // 'go get' is expected to do this, unlike other commands. modload.AllowMissingModuleImports() + // 'go get' no longer builds or installs packages, so there's nothing to do + // if there's no go.mod file. + // TODO(#40775): make modload.Init return ErrNoModRoot instead of exiting. + // We could handle that here by printing a different message. + modload.Init() + if !modload.HasModRoot() { + base.Fatalf("go: go.mod file not found in current directory or any parent directory.\n" + + "\t'go get' is no longer supported outside a module.\n" + + "\tTo build and install a command, use 'go install' with a version,\n" + + "\tlike 'go install example.com/cmd@latest'\n" + + "\tFor more information, see https://golang.org/doc/go-get-install-deprecation\n" + + "\tor run 'go help get' or 'go help install'.") + } + queries := parseArgs(ctx, args) r := newResolver(ctx, queries) @@ -356,74 +370,12 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } r.checkPackageProblems(ctx, pkgPatterns) - // We've already downloaded modules (and identified direct and indirect - // dependencies) by loading packages in findAndUpgradeImports. - // So if -d is set, we're done after the module work. - // - // Otherwise, we need to build and install the packages matched by - // command line arguments. - // Note that 'go get -u' without arguments is equivalent to - // 'go get -u .', so we'll typically build the package in the current - // directory. - if !*getD && len(pkgPatterns) > 0 { - work.BuildInit() - - pkgOpts := load.PackageOpts{ModResolveTests: *getT} - var pkgs []*load.Package - for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, pkgPatterns) { - if pkg.Error != nil { - var noGo *load.NoGoError - if errors.As(pkg.Error.Err, &noGo) { - if m := modload.PackageModule(pkg.ImportPath); m.Path == pkg.ImportPath { - // pkg is at the root of a module, and doesn't exist with the current - // build tags. Probably the user just wanted to change the version of - // that module — not also build the package — so suppress the error. - // (See https://golang.org/issue/33526.) - continue - } - } - } - pkgs = append(pkgs, pkg) - } - load.CheckPackageErrors(pkgs) - - haveExternalExe := false - for _, pkg := range pkgs { - if pkg.Name == "main" && pkg.Module != nil && pkg.Module.Path != modload.Target.Path { - haveExternalExe = true - break - } - } - if haveExternalExe { - fmt.Fprint(os.Stderr, "go get: installing executables with 'go get' in module mode is deprecated.") - var altMsg string - if modload.HasModRoot() { - altMsg = ` - To adjust and download dependencies of the current module, use 'go get -d'. - To install using requirements of the current module, use 'go install'. - To install ignoring the current module, use 'go install' with a version, - like 'go install example.com/cmd@latest'. -` - } else { - altMsg = "\n\tUse 'go install pkg@version' instead.\n" - } - fmt.Fprint(os.Stderr, altMsg) - fmt.Fprintf(os.Stderr, "\tFor more information, see https://golang.org/doc/go-get-install-deprecation\n\tor run 'go help get' or 'go help install'.\n") - } - - work.InstallPackages(ctx, pkgPatterns, pkgs) - } - - if !modload.HasModRoot() { - return - } - // Everything succeeded. Update go.mod. oldReqs := reqsFromGoMod(modload.ModFile()) - modload.AllowWriteGoMod() - modload.WriteGoMod(ctx) - modload.DisallowWriteGoMod() + if err := modload.WriteGoMod(ctx); err != nil { + base.Fatalf("go: %v", err) + } newReqs := reqsFromGoMod(modload.ModFile()) r.reportChanges(oldReqs, newReqs) @@ -440,7 +392,7 @@ func parseArgs(ctx context.Context, rawArgs []string) []*query { for _, arg := range search.CleanPatterns(rawArgs) { q, err := newQuery(arg) if err != nil { - base.Errorf("go get: %v", err) + base.Errorf("go: %v", err) continue } @@ -455,11 +407,11 @@ func parseArgs(ctx context.Context, rawArgs []string) []*query { // if the argument has no version and either has no slash or refers to an existing file. if strings.HasSuffix(q.raw, ".go") && q.rawVersion == "" { if !strings.Contains(q.raw, "/") { - base.Errorf("go get %s: arguments must be package or module paths", q.raw) + base.Errorf("go: %s: arguments must be package or module paths", q.raw) continue } if fi, err := os.Stat(q.raw); err == nil && !fi.IsDir() { - base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", q.raw) + base.Errorf("go: %s exists as a file, but 'go get' requires package arguments", q.raw) continue } } @@ -649,7 +601,7 @@ func (r *resolver) matchInModule(ctx context.Context, pattern string, m module.V err error } - e := r.matchInModuleCache.Do(key{pattern, m}, func() interface{} { + e := r.matchInModuleCache.Do(key{pattern, m}, func() any { match := modload.MatchInModule(ctx, pattern, m, imports.AnyTags()) if len(match.Errs) > 0 { return entry{match.Pkgs, match.Errs[0]} @@ -675,7 +627,9 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { if !q.isWildcard() { q.pathOnce(q.pattern, func() pathSet { - if modload.HasModRoot() && q.pattern == modload.Target.Path { + hasModRoot := modload.HasModRoot() + if hasModRoot && modload.MainModules.Contains(q.pattern) { + v := module.Version{Path: q.pattern} // The user has explicitly requested to downgrade their own module to // version "none". This is not an entirely unreasonable request: it // could plausibly mean “downgrade away everything that depends on any @@ -686,7 +640,7 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { // However, neither of those behaviors would be consistent with the // plain meaning of the query. To try to reduce confusion, reject the // query explicitly. - return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) + return errSet(&modload.QueryMatchesMainModulesError{MainModules: []module.Version{v}, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: q.pattern, Version: "none"}} @@ -698,8 +652,8 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { continue } q.pathOnce(curM.Path, func() pathSet { - if modload.HasModRoot() && curM == modload.Target { - return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) + if modload.HasModRoot() && curM.Version == "" && modload.MainModules.Contains(curM.Path) { + return errSet(&modload.QueryMatchesMainModulesError{MainModules: []module.Version{curM}, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: curM.Path, Version: "none"}} }) @@ -718,28 +672,38 @@ func (r *resolver) performLocalQueries(ctx context.Context) { // Absolute paths like C:\foo and relative paths like ../foo... are // restricted to matching packages in the main module. - pkgPattern := modload.DirImportPath(ctx, q.pattern) + pkgPattern, mainModule := modload.MainModules.DirImportPath(ctx, q.pattern) if pkgPattern == "." { - return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot())) + modload.MustHaveModRoot() + var modRoots []string + for _, m := range modload.MainModules.Versions() { + modRoots = append(modRoots, modload.MainModules.ModRoot(m)) + } + var plural string + if len(modRoots) != 1 { + plural = "s" + } + return errSet(fmt.Errorf("%s%s is not within module%s rooted at %s", q.pattern, absDetail, plural, strings.Join(modRoots, ", "))) } - match := modload.MatchInModule(ctx, pkgPattern, modload.Target, imports.AnyTags()) + match := modload.MatchInModule(ctx, pkgPattern, mainModule, imports.AnyTags()) if len(match.Errs) > 0 { return pathSet{err: match.Errs[0]} } if len(match.Pkgs) == 0 { if q.raw == "" || q.raw == "." { - return errSet(fmt.Errorf("no package in current directory")) + return errSet(fmt.Errorf("no package to get in current directory")) } if !q.isWildcard() { - return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.ModRoot())) + modload.MustHaveModRoot() + return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.MainModules.ModRoot(mainModule))) } search.WarnUnmatched([]*search.Match{match}) return pathSet{} } - return pathSet{pkgMods: []module.Version{modload.Target}} + return pathSet{pkgMods: []module.Version{mainModule}} }) } } @@ -789,11 +753,12 @@ func (r *resolver) queryWildcard(ctx context.Context, q *query) { return pathSet{} } - if curM.Path == modload.Target.Path && !versionOkForMainModule(q.version) { + if modload.MainModules.Contains(curM.Path) && !versionOkForMainModule(q.version) { if q.matchesPath(curM.Path) { - return errSet(&modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + return errSet(&modload.QueryMatchesMainModulesError{ + MainModules: []module.Version{curM}, + Pattern: q.pattern, + Query: q.version, }) } @@ -928,7 +893,7 @@ func (r *resolver) checkWildcardVersions(ctx context.Context) { // curM at its original version contains a path matching q.pattern, // but at rev.Version it does not, so (somewhat paradoxically) if // we changed the version of curM it would no longer match the query. - var version interface{} = m + var version any = m if rev.Version != q.version { version = fmt.Sprintf("%s@%s (%s)", m.Path, q.version, m.Version) } @@ -1159,8 +1124,8 @@ func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPack } opts.AllowPackage = func(ctx context.Context, path string, m module.Version) error { - if m.Path == "" || m == modload.Target { - // Packages in the standard library and main module are already at their + if m.Path == "" || m.Version == "" { + // Packages in the standard library and main modules are already at their // latest (and only) available versions. return nil } @@ -1327,7 +1292,7 @@ func (r *resolver) applyUpgrades(ctx context.Context, upgrades []pathSet) (chang var tentative []module.Version for _, cs := range upgrades { if cs.err != nil { - base.Errorf("go get: %v", cs.err) + base.Errorf("go: %v", cs.err) continue } @@ -1370,11 +1335,11 @@ func (r *resolver) disambiguate(cs pathSet) (filtered pathSet, isPackage bool, m continue } - if m.Path == modload.Target.Path { - if m.Version == modload.Target.Version { + if modload.MainModules.Contains(m.Path) { + if m.Version == "" { return pathSet{}, true, m, true } - // The main module can only be set to its own version. + // A main module can only be set to its own version. continue } @@ -1720,13 +1685,13 @@ func (r *resolver) reportChanges(oldReqs, newReqs []module.Version) { }) for _, c := range sortedChanges { if c.old == "" { - fmt.Fprintf(os.Stderr, "go get: added %s %s\n", c.path, c.new) + fmt.Fprintf(os.Stderr, "go: added %s %s\n", c.path, c.new) } else if c.new == "none" || c.new == "" { - fmt.Fprintf(os.Stderr, "go get: removed %s %s\n", c.path, c.old) + fmt.Fprintf(os.Stderr, "go: removed %s %s\n", c.path, c.old) } else if semver.Compare(c.new, c.old) > 0 { - fmt.Fprintf(os.Stderr, "go get: upgraded %s %s => %s\n", c.path, c.old, c.new) + fmt.Fprintf(os.Stderr, "go: upgraded %s %s => %s\n", c.path, c.old, c.new) } else { - fmt.Fprintf(os.Stderr, "go get: downgraded %s %s => %s\n", c.path, c.old, c.new) + fmt.Fprintf(os.Stderr, "go: downgraded %s %s => %s\n", c.path, c.old, c.new) } } @@ -1744,10 +1709,11 @@ func (r *resolver) resolve(q *query, m module.Version) { panic("internal error: resolving a module.Version with an empty path") } - if m.Path == modload.Target.Path && m.Version != modload.Target.Version { - reportError(q, &modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + if modload.MainModules.Contains(m.Path) && m.Version != "" { + reportError(q, &modload.QueryMatchesMainModulesError{ + MainModules: []module.Version{{Path: m.Path}}, + Pattern: q.pattern, + Query: q.version, }) return } @@ -1775,7 +1741,7 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi resolved := make([]module.Version, 0, len(r.resolvedVersion)) for mPath, rv := range r.resolvedVersion { - if mPath != modload.Target.Path { + if !modload.MainModules.Contains(mPath) { resolved = append(resolved, module.Version{Path: mPath, Version: rv.version}) } } @@ -1784,7 +1750,7 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi if err != nil { var constraint *modload.ConstraintError if !errors.As(err, &constraint) { - base.Errorf("go get: %v", err) + base.Errorf("go: %v", err) return false } @@ -1796,7 +1762,7 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi return rv.reason.ResolvedString(module.Version{Path: m.Path, Version: rv.version}) } for _, c := range constraint.Conflicts { - base.Errorf("go get: %v requires %v, not %v", reason(c.Source), c.Dep, reason(c.Constraint)) + base.Errorf("go: %v requires %v, not %v", reason(c.Source), c.Dep, reason(c.Constraint)) } return false } diff --git a/libgo/go/cmd/go/internal/modget/query.go b/libgo/go/cmd/go/internal/modget/query.go index 1a5a60f7eb9..887cb51b317 100644 --- a/libgo/go/cmd/go/internal/modget/query.go +++ b/libgo/go/cmd/go/internal/modget/query.go @@ -192,9 +192,9 @@ func (q *query) validate() error { // TODO(bcmills): "all@none" seems like a totally reasonable way to // request that we remove all module requirements, leaving only the main // module and standard library. Perhaps we should implement that someday. - return &modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + return &modload.QueryUpgradesAllError{ + MainModules: modload.MainModules.Versions(), + Query: q.version, } } } @@ -284,21 +284,21 @@ func reportError(q *query, err error) { patternRE := regexp.MustCompile("(?m)(?:[ \t(\"`]|^)" + regexp.QuoteMeta(q.pattern) + "(?:[ @:;)\"`]|$)") if patternRE.MatchString(errStr) { if q.rawVersion == "" { - base.Errorf("go get: %s", errStr) + base.Errorf("go: %s", errStr) return } versionRE := regexp.MustCompile("(?m)(?:[ @(\"`]|^)" + regexp.QuoteMeta(q.version) + "(?:[ :;)\"`]|$)") if versionRE.MatchString(errStr) { - base.Errorf("go get: %s", errStr) + base.Errorf("go: %s", errStr) return } } if qs := q.String(); qs != "" { - base.Errorf("go get %s: %s", qs, errStr) + base.Errorf("go: %s: %s", qs, errStr) } else { - base.Errorf("go get: %s", errStr) + base.Errorf("go: %s", errStr) } } diff --git a/libgo/go/cmd/go/internal/modload/build.go b/libgo/go/cmd/go/internal/modload/build.go index becf6b87f80..bfc73cc2f9a 100644 --- a/libgo/go/cmd/go/internal/modload/build.go +++ b/libgo/go/cmd/go/internal/modload/build.go @@ -5,7 +5,6 @@ package modload import ( - "bytes" "context" "encoding/hex" "errors" @@ -80,7 +79,7 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { v string ok bool ) - if rs.depth == lazy { + if rs.pruning == pruned { v, ok = rs.rootSelected(path) } if !ok { @@ -212,20 +211,20 @@ func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) { // in rs (which may be nil to indicate that m was not loaded from a requirement // graph). func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode) *modinfo.ModulePublic { - if m == Target { + if m.Version == "" && MainModules.Contains(m.Path) { info := &modinfo.ModulePublic{ Path: m.Path, Version: m.Version, Main: true, } - if v, ok := rawGoVersion.Load(Target); ok { + if v, ok := rawGoVersion.Load(m); ok { info.GoVersion = v.(string) } else { panic("internal error: GoVersion not set for main module") } - if HasModRoot() { - info.Dir = ModRoot() - info.GoMod = ModFilePath() + if modRoot := MainModules.ModRoot(m); modRoot != "" { + info.Dir = modRoot + info.GoMod = modFilePath(modRoot) } return info } @@ -322,7 +321,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li if filepath.IsAbs(r.Path) { info.Replace.Dir = r.Path } else { - info.Replace.Dir = filepath.Join(ModRoot(), r.Path) + info.Replace.Dir = filepath.Join(replaceRelativeTo(), r.Path) } info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") } @@ -336,116 +335,33 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li return info } -// PackageBuildInfo returns a string containing module version information -// for modules providing packages named by path and deps. path and deps must -// name packages that were resolved successfully with LoadPackages. -func PackageBuildInfo(path string, deps []string) string { - if isStandardImportPath(path) || !Enabled() { - return "" - } - - target := mustFindModule(loaded, path, path) - mdeps := make(map[module.Version]bool) - for _, dep := range deps { - if !isStandardImportPath(dep) { - mdeps[mustFindModule(loaded, path, dep)] = true - } - } - var mods []module.Version - delete(mdeps, target) - for mod := range mdeps { - mods = append(mods, mod) - } - module.Sort(mods) - - var buf bytes.Buffer - fmt.Fprintf(&buf, "path\t%s\n", path) - - writeEntry := func(token string, m module.Version) { - mv := m.Version - if mv == "" { - mv = "(devel)" - } - fmt.Fprintf(&buf, "%s\t%s\t%s", token, m.Path, mv) - if r := Replacement(m); r.Path == "" { - fmt.Fprintf(&buf, "\t%s\n", modfetch.Sum(m)) - } else { - fmt.Fprintf(&buf, "\n=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r)) - } - } - - writeEntry("mod", target) - for _, mod := range mods { - writeEntry("dep", mod) - } - - return buf.String() -} - -// mustFindModule is like findModule, but it calls base.Fatalf if the -// module can't be found. -// -// TODO(jayconrod): remove this. Callers should use findModule and return -// errors instead of relying on base.Fatalf. -func mustFindModule(ld *loader, target, path string) module.Version { - pkg, ok := ld.pkgCache.Get(path).(*loadPkg) - if ok { - if pkg.err != nil { - base.Fatalf("build %v: cannot load %v: %v", target, path, pkg.err) - } - return pkg.mod - } - - if path == "command-line-arguments" { - return Target - } - - base.Fatalf("build %v: cannot find module for path %v", target, path) - panic("unreachable") -} - // findModule searches for the module that contains the package at path. // If the package was loaded, its containing module and true are returned. -// Otherwise, module.Version{} and false are returend. +// Otherwise, module.Version{} and false are returned. func findModule(ld *loader, path string) (module.Version, bool) { if pkg, ok := ld.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} } - if path == "command-line-arguments" { - return Target, true - } return module.Version{}, false } func ModInfoProg(info string, isgccgo bool) []byte { - // Inject a variable with the debug information as runtime.modinfo, - // but compile it in package main so that it is specific to the binary. - // The variable must be a literal so that it will have the correct value - // before the initializer for package main runs. - // - // The runtime startup code refers to the variable, which keeps it live - // in all binaries. - // - // Note: we use an alternate recipe below for gccgo (based on an - // init function) due to the fact that gccgo does not support - // applying a "//go:linkname" directive to a variable. This has - // drawbacks in that other packages may want to look at the module - // info in their init functions (see issue 29628), which won't - // work for gccgo. See also issue 30344. - - if !isgccgo { - return []byte(fmt.Sprintf(`package main -import _ "unsafe" -//go:linkname __set_modinfo__ runtime.setmodinfo -func __set_modinfo__(string) -func init() { __set_modinfo__(%q) } - `, string(infoStart)+info+string(infoEnd))) - } else { + // Inject an init function to set runtime.modinfo. + // This is only used for gccgo - with gc we hand the info directly to the linker. + // The init function has the drawback that packages may want to + // look at the module info in their init functions (see issue 29628), + // which won't work. See also issue 30344. + if isgccgo { return []byte(fmt.Sprintf(`package main import _ "unsafe" //go:linkname __set_debug_modinfo__ runtime.setmodinfo func __set_debug_modinfo__(string) func init() { __set_debug_modinfo__(%q) } -`, string(infoStart)+info+string(infoEnd))) +`, ModInfoData(info))) } + return nil +} + +func ModInfoData(info string) []byte { + return []byte(string(infoStart) + info + string(infoEnd)) } diff --git a/libgo/go/cmd/go/internal/modload/buildlist.go b/libgo/go/cmd/go/internal/modload/buildlist.go index bf695673167..6f9072c8c48 100644 --- a/libgo/go/cmd/go/internal/modload/buildlist.go +++ b/libgo/go/cmd/go/internal/modload/buildlist.go @@ -30,18 +30,25 @@ func capVersionSlice(s []module.Version) []module.Version { // A Requirements represents a logically-immutable set of root module requirements. type Requirements struct { - // depth is the depth at which the requirement graph is computed. + // pruning is the pruning at which the requirement graph is computed. // - // If eager, the graph includes all transitive requirements regardless of depth. + // If unpruned, the graph includes all transitive requirements regardless + // of whether the requiring module supports pruning. // - // If lazy, the graph includes only the root modules, the explicit + // If pruned, the graph includes only the root modules, the explicit // requirements of those root modules, and the transitive requirements of only - // the *non-lazy* root modules. - depth modDepth - - // rootModules is the set of module versions explicitly required by the main - // module, sorted and capped to length. It may contain duplicates, and may - // contain multiple versions for a given module path. + // the root modules that do not support pruning. + // + // If workspace, the graph includes only the workspace modules, the explicit + // requirements of the workspace modules, and the transitive requirements of + // the workspace modules that do not support pruning. + pruning modPruning + + // rootModules is the set of root modules of the graph, sorted and capped to + // length. It may contain duplicates, and may contain multiple versions for a + // given module path. The root modules of the groph are the set of main + // modules in workspace mode, and the main module's direct requirements + // outside workspace mode. rootModules []module.Version maxRootVersion map[string]string @@ -97,10 +104,23 @@ var requirements *Requirements // // If vendoring is in effect, the caller must invoke initVendor on the returned // *Requirements before any other method. -func newRequirements(depth modDepth, rootModules []module.Version, direct map[string]bool) *Requirements { +func newRequirements(pruning modPruning, rootModules []module.Version, direct map[string]bool) *Requirements { + if pruning == workspace { + return &Requirements{ + pruning: pruning, + rootModules: capVersionSlice(rootModules), + maxRootVersion: nil, + direct: direct, + } + } + + if workFilePath != "" && pruning != workspace { + panic("in workspace mode, but pruning is not workspace in newRequirements") + } + for i, m := range rootModules { - if m == Target { - panic(fmt.Sprintf("newRequirements called with untrimmed build list: rootModules[%v] is Target", i)) + if m.Version == "" && MainModules.Contains(m.Path) { + panic(fmt.Sprintf("newRequirements called with untrimmed build list: rootModules[%v] is a main module", i)) } if m.Path == "" || m.Version == "" { panic(fmt.Sprintf("bad requirement: rootModules[%v] = %v", i, m)) @@ -114,7 +134,7 @@ func newRequirements(depth modDepth, rootModules []module.Version, direct map[st } rs := &Requirements{ - depth: depth, + pruning: pruning, rootModules: capVersionSlice(rootModules), maxRootVersion: make(map[string]string, len(rootModules)), direct: direct, @@ -135,13 +155,18 @@ func newRequirements(depth modDepth, rootModules []module.Version, direct map[st func (rs *Requirements) initVendor(vendorList []module.Version) { rs.graphOnce.Do(func() { mg := &ModuleGraph{ - g: mvs.NewGraph(cmpVersion, []module.Version{Target}), + g: mvs.NewGraph(cmpVersion, MainModules.Versions()), } - if rs.depth == lazy { - // The roots of a lazy module should already include every module in the - // vendor list, because the vendored modules are the same as those - // maintained as roots by the lazy loading “import invariant”. + if MainModules.Len() != 1 { + panic("There should be exactly one main module in Vendor mode.") + } + mainModule := MainModules.Versions()[0] + + if rs.pruning == pruned { + // The roots of a pruned module should already include every module in the + // vendor list, because the vendored modules are the same as those needed + // for graph pruning. // // Just to be sure, we'll double-check that here. inconsistent := false @@ -156,9 +181,9 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { } // Now we can treat the rest of the module graph as effectively “pruned - // out”, like a more aggressive version of lazy loading: in vendor mode, - // the root requirements *are* the complete module graph. - mg.g.Require(Target, rs.rootModules) + // out”, as though we are viewing the main module from outside: in vendor + // mode, the root requirements *are* the complete module graph. + mg.g.Require(mainModule, rs.rootModules) } else { // The transitive requirements of the main module are not in general available // from the vendor directory, and we don't actually know how we got from @@ -170,7 +195,7 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { // graph, but still distinguishes between direct and indirect // dependencies. vendorMod := module.Version{Path: "vendor/modules.txt", Version: ""} - mg.g.Require(Target, append(rs.rootModules, vendorMod)) + mg.g.Require(mainModule, append(rs.rootModules, vendorMod)) mg.g.Require(vendorMod, vendorList) } @@ -182,8 +207,8 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { // path, or the zero module.Version and ok=false if the module is not a root // dependency. func (rs *Requirements) rootSelected(path string) (version string, ok bool) { - if path == Target.Path { - return Target.Version, true + if MainModules.Contains(path) { + return "", true } if v, ok := rs.maxRootVersion[path]; ok { return v, true @@ -197,7 +222,7 @@ func (rs *Requirements) rootSelected(path string) (version string, ok bool) { // selection. func (rs *Requirements) hasRedundantRoot() bool { for i, m := range rs.rootModules { - if m.Path == Target.Path || (i > 0 && m.Path == rs.rootModules[i-1].Path) { + if MainModules.Contains(m.Path) || (i > 0 && m.Path == rs.rootModules[i-1].Path) { return true } } @@ -214,7 +239,7 @@ func (rs *Requirements) hasRedundantRoot() bool { // returns a non-nil error of type *mvs.BuildListError. func (rs *Requirements) Graph(ctx context.Context) (*ModuleGraph, error) { rs.graphOnce.Do(func() { - mg, mgErr := readModGraph(ctx, rs.depth, rs.rootModules) + mg, mgErr := readModGraph(ctx, rs.pruning, rs.rootModules) rs.graph.Store(cachedGraph{mg, mgErr}) }) cached := rs.graph.Load().(cachedGraph) @@ -230,7 +255,7 @@ func (rs *Requirements) IsDirect(path string) bool { // A ModuleGraph represents the complete graph of module dependencies // of a main module. // -// If the main module is lazily loaded, the graph does not include +// If the main module supports module graph pruning, the graph does not include // transitive dependencies of non-root (implicit) dependencies. type ModuleGraph struct { g *mvs.Graph @@ -254,8 +279,16 @@ var readModGraphDebugOnce sync.Once // // Unlike LoadModGraph, readModGraph does not attempt to diagnose or update // inconsistent roots. -func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) (*ModuleGraph, error) { - if depth == lazy { +func readModGraph(ctx context.Context, pruning modPruning, roots []module.Version) (*ModuleGraph, error) { + if pruning == pruned { + // Enable diagnostics for lazy module loading + // (https://golang.org/ref/mod#lazy-loading) only if the module graph is + // pruned. + // + // In unpruned modules,we load the module graph much more aggressively (in + // order to detect inconsistencies that wouldn't be feasible to spot-check), + // so it wouldn't be useful to log when that occurs (because it happens in + // normal operation all the time). readModGraphDebugOnce.Do(func() { for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") { switch f { @@ -274,21 +307,26 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( mu sync.Mutex // guards mg.g and hasError during loading hasError bool mg = &ModuleGraph{ - g: mvs.NewGraph(cmpVersion, []module.Version{Target}), + g: mvs.NewGraph(cmpVersion, MainModules.Versions()), } ) - mg.g.Require(Target, roots) + if pruning != workspace { + if inWorkspaceMode() { + panic("pruning is not workspace in workspace mode") + } + mg.g.Require(MainModules.mustGetSingleMainModule(), roots) + } var ( - loadQueue = par.NewQueue(runtime.GOMAXPROCS(0)) - loadingEager sync.Map // module.Version → nil; the set of modules that have been or are being loaded via eager roots + loadQueue = par.NewQueue(runtime.GOMAXPROCS(0)) + loadingUnpruned sync.Map // module.Version → nil; the set of modules that have been or are being loaded via roots that do not support pruning ) // loadOne synchronously loads the explicit requirements for module m. // It does not load the transitive requirements of m even if the go version in - // m's go.mod file indicates eager loading. + // m's go.mod file indicates that it supports graph pruning. loadOne := func(m module.Version) (*modFileSummary, error) { - cached := mg.loadCache.Do(m, func() interface{} { + cached := mg.loadCache.Do(m, func() any { summary, err := goModSummary(m) mu.Lock() @@ -305,16 +343,16 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( return cached.summary, cached.err } - var enqueue func(m module.Version, depth modDepth) - enqueue = func(m module.Version, depth modDepth) { + var enqueue func(m module.Version, pruning modPruning) + enqueue = func(m module.Version, pruning modPruning) { if m.Version == "none" { return } - if depth == eager { - if _, dup := loadingEager.LoadOrStore(m, nil); dup { - // m has already been enqueued for loading. Since eager loading may - // follow cycles in the the requirement graph, we need to return early + if pruning == unpruned { + if _, dup := loadingUnpruned.LoadOrStore(m, nil); dup { + // m has already been enqueued for loading. Since unpruned loading may + // follow cycles in the requirement graph, we need to return early // to avoid making the load queue infinitely long. return } @@ -326,24 +364,74 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( return // findError will report the error later. } - // If the version in m's go.mod file implies eager loading, then we cannot - // assume that the explicit requirements of m (added by loadOne) are - // sufficient to build the packages it contains. We must load its full + // If the version in m's go.mod file does not support pruning, then we + // cannot assume that the explicit requirements of m (added by loadOne) + // are sufficient to build the packages it contains. We must load its full // transitive dependency graph to be sure that we see all relevant // dependencies. - if depth == eager || summary.depth == eager { + if pruning != pruned || summary.pruning == unpruned { + nextPruning := summary.pruning + if pruning == unpruned { + nextPruning = unpruned + } for _, r := range summary.require { - enqueue(r, eager) + enqueue(r, nextPruning) } } }) } for _, m := range roots { - enqueue(m, depth) + enqueue(m, pruning) } <-loadQueue.Idle() + // Reload any dependencies of the main modules which are not + // at their selected versions at workspace mode, because the + // requirements don't accurately reflect the transitive imports. + if pruning == workspace { + // hasDepsInAll contains the set of modules that need to be loaded + // at workspace pruning because any of their dependencies may + // provide packages in all. + hasDepsInAll := make(map[string]bool) + seen := map[module.Version]bool{} + for _, m := range roots { + hasDepsInAll[m.Path] = true + seen[m] = true + } + // This loop will terminate because it will call enqueue on each version of + // each dependency of the modules in hasDepsInAll at most once (and only + // calls enqueue on successively increasing versions of each dependency). + for { + needsEnqueueing := map[module.Version]bool{} + for p := range hasDepsInAll { + m := module.Version{Path: p, Version: mg.g.Selected(p)} + reqs, ok := mg.g.RequiredBy(m) + if !ok { + needsEnqueueing[m] = true + continue + } + for _, r := range reqs { + s := module.Version{Path: r.Path, Version: mg.g.Selected(r.Path)} + if cmpVersion(s.Version, r.Version) > 0 && !seen[s] { + needsEnqueueing[s] = true + } + } + } + // add all needs enqueueing to paths we care about + if len(needsEnqueueing) == 0 { + break + } + + for p := range needsEnqueueing { + enqueue(p, workspace) + seen[p] = true + hasDepsInAll[p.Path] = true + } + <-loadQueue.Idle() + } + } + if hasError { return mg, mg.findError() } @@ -351,8 +439,7 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( } // RequiredBy returns the dependencies required by module m in the graph, -// or ok=false if module m's dependencies are not relevant (such as if they -// are pruned out by lazy loading). +// or ok=false if module m's dependencies are pruned out. // // The caller must not modify the returned slice, but may safely append to it // and may rely on it not to be modified. @@ -404,7 +491,12 @@ func (mg *ModuleGraph) findError() error { } func (mg *ModuleGraph) allRootsSelected() bool { - roots, _ := mg.g.RequiredBy(Target) + var roots []module.Version + if inWorkspaceMode() { + roots = MainModules.Versions() + } else { + roots, _ = mg.g.RequiredBy(MainModules.mustGetSingleMainModule()) + } for _, m := range roots { if mg.Selected(m.Path) != m.Version { return false @@ -427,12 +519,12 @@ func LoadModGraph(ctx context.Context, goVersion string) *ModuleGraph { rs := LoadModFile(ctx) if goVersion != "" { - depth := modDepthFromGoVersion(goVersion) - if depth == eager && rs.depth != eager { + pruning := pruningForGoVersion(goVersion) + if pruning == unpruned && rs.pruning != unpruned { // Use newRequirements instead of convertDepth because convertDepth // also updates roots; here, we want to report the unmodified roots // even though they may seem inconsistent. - rs = newRequirements(eager, rs.rootModules, rs.direct) + rs = newRequirements(unpruned, rs.rootModules, rs.direct) } mg, err := rs.Graph(ctx) @@ -447,7 +539,8 @@ func LoadModGraph(ctx context.Context, goVersion string) *ModuleGraph { base.Fatalf("go: %v", err) } - commitRequirements(ctx, modFileGoVersion(), rs) + requirements = rs + return mg } @@ -455,9 +548,8 @@ func LoadModGraph(ctx context.Context, goVersion string) *ModuleGraph { // // If the complete graph reveals that some root of rs is not actually the // selected version of its path, expandGraph computes a new set of roots that -// are consistent. (When lazy loading is implemented, this may result in -// upgrades to other modules due to requirements that were previously pruned -// out.) +// are consistent. (With a pruned module graph, this may result in upgrades to +// other modules due to requirements that were previously pruned out.) // // expandGraph returns the updated roots, along with the module graph loaded // from those roots and any error encountered while loading that graph. @@ -473,9 +565,9 @@ func expandGraph(ctx context.Context, rs *Requirements) (*Requirements, *ModuleG if !mg.allRootsSelected() { // The roots of rs are not consistent with the rest of the graph. Update - // them. In an eager module this is a no-op for the build list as a whole — + // them. In an unpruned module this is a no-op for the build list as a whole — // it just promotes what were previously transitive requirements to be - // roots — but in a lazy module it may pull in previously-irrelevant + // roots — but in a pruned module it may pull in previously-irrelevant // transitive dependencies. newRS, rsErr := updateRoots(ctx, rs.direct, rs, nil, nil, false) @@ -513,7 +605,7 @@ func EditBuildList(ctx context.Context, add, mustSelect []module.Version) (chang if err != nil { return false, err } - commitRequirements(ctx, modFileGoVersion(), rs) + requirements = rs return changed, err } @@ -544,23 +636,45 @@ type Conflict struct { // tidyRoots trims the root dependencies to the minimal requirements needed to // both retain the same versions of all packages in pkgs and satisfy the -// lazy loading invariants (if applicable). +// graph-pruning invariants (if applicable). func tidyRoots(ctx context.Context, rs *Requirements, pkgs []*loadPkg) (*Requirements, error) { - if rs.depth == eager { - return tidyEagerRoots(ctx, rs.direct, pkgs) + mainModule := MainModules.mustGetSingleMainModule() + if rs.pruning == unpruned { + return tidyUnprunedRoots(ctx, mainModule, rs.direct, pkgs) } - return tidyLazyRoots(ctx, rs.direct, pkgs) + return tidyPrunedRoots(ctx, mainModule, rs.direct, pkgs) } func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { - if rs.depth == eager { - return updateEagerRoots(ctx, direct, rs, add) + switch rs.pruning { + case unpruned: + return updateUnprunedRoots(ctx, direct, rs, add) + case pruned: + return updatePrunedRoots(ctx, direct, rs, pkgs, add, rootsImported) + case workspace: + return updateWorkspaceRoots(ctx, rs, add) + default: + panic(fmt.Sprintf("unsupported pruning mode: %v", rs.pruning)) + } +} + +func updateWorkspaceRoots(ctx context.Context, rs *Requirements, add []module.Version) (*Requirements, error) { + if len(add) != 0 { + // add should be empty in workspace mode because workspace mode implies + // -mod=readonly, which in turn implies no new requirements. The code path + // that would result in add being non-empty returns an error before it + // reaches this point: The set of modules to add comes from + // resolveMissingImports, which in turn resolves each package by calling + // queryImport. But queryImport explicitly checks for -mod=readonly, and + // return an error. + panic("add is not empty") } - return updateLazyRoots(ctx, direct, rs, pkgs, add, rootsImported) + return rs, nil } -// tidyLazyRoots returns a minimal set of root requirements that maintains the -// "lazy loading" invariants of the go.mod file for the given packages: +// tidyPrunedRoots returns a minimal set of root requirements that maintains the +// invariants of the go.mod file needed to support graph pruning for the given +// packages: // // 1. For each package marked with pkgInAll, the module path that provided that // package is included as a root. @@ -568,16 +682,16 @@ func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, // selected at the same version or is upgraded by the dependencies of a // root. // -// If any module that provided a package has been upgraded above its previous, +// If any module that provided a package has been upgraded above its previous // version, the caller may need to reload and recompute the package graph. // // To ensure that the loading process eventually converges, the caller should // add any needed roots from the tidy root set (without removing existing untidy // roots) until the set of roots has converged. -func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { +func tidyPrunedRoots(ctx context.Context, mainModule module.Version, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { var ( roots []module.Version - pathIncluded = map[string]bool{Target.Path: true} + pathIncluded = map[string]bool{mainModule.Path: true} ) // We start by adding roots for every package in "all". // @@ -605,7 +719,7 @@ func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) queued[pkg] = true } module.Sort(roots) - tidy := newRequirements(lazy, roots, direct) + tidy := newRequirements(pruned, roots, direct) for len(queue) > 0 { roots = tidy.rootModules @@ -641,7 +755,7 @@ func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) if len(roots) > len(tidy.rootModules) { module.Sort(roots) - tidy = newRequirements(lazy, roots, tidy.direct) + tidy = newRequirements(pruned, roots, tidy.direct) } } @@ -652,8 +766,8 @@ func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) return tidy, nil } -// updateLazyRoots returns a set of root requirements that maintains the “lazy -// loading” invariants of the go.mod file: +// updatePrunedRoots returns a set of root requirements that maintains the +// invariants of the go.mod file needed to support graph pruning: // // 1. The selected version of the module providing each package marked with // either pkgInAll or pkgIsRoot is included as a root. @@ -670,7 +784,7 @@ func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) // The packages in pkgs are assumed to have been loaded from either the roots of // rs or the modules selected in the graph of rs. // -// The above invariants together imply the “lazy loading” invariants for the +// The above invariants together imply the graph-pruning invariants for the // go.mod file: // // 1. (The import invariant.) Every module that provides a package transitively @@ -690,13 +804,13 @@ func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) // it requires explicitly. This invariant is left up to the caller, who must // not load packages from outside the module graph but may add roots to the // graph, but is facilited by (3). If the caller adds roots to the graph in -// order to resolve missing packages, then updateLazyRoots will retain them, +// order to resolve missing packages, then updatePrunedRoots will retain them, // the selected versions of those roots cannot regress, and they will // eventually be written back to the main module's go.mod file. // // (See https://golang.org/design/36460-lazy-module-loading#invariants for more // detail.) -func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { +func updatePrunedRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { roots := rs.rootModules rootsUpgraded := false @@ -717,11 +831,11 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen // pkg is transitively imported by a package or test in the main module. // We need to promote the module that maintains it to a root: if some // other module depends on the main module, and that other module also - // uses lazy loading, it will expect to find all of our transitive - // dependencies by reading just our go.mod file, not the go.mod files of - // everything we depend on. + // uses a pruned module graph, it will expect to find all of our + // transitive dependencies by reading just our go.mod file, not the go.mod + // files of everything we depend on. // - // (This is the “import invariant” that makes lazy loading possible.) + // (This is the “import invariant” that makes graph pruning possible.) case rootsImported && pkg.flags.has(pkgFromRoot): // pkg is a transitive dependency of some root, and we are treating the @@ -732,17 +846,18 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen // it matches a command-line argument.) We want future invocations of the // 'go' command — such as 'go test' on the same package — to continue to // use the same versions of its dependencies that we are using right now. - // So we need to bring this package's dependencies inside the lazy-loading - // horizon. + // So we need to bring this package's dependencies inside the pruned + // module graph. // // Making the module containing this package a root of the module graph - // does exactly that: if the module containing the package is lazy it - // should satisfy the import invariant itself, so all of its dependencies - // should be in its go.mod file, and if the module containing the package - // is eager then if we make it a root we will load all of its transitive - // dependencies into the module graph. + // does exactly that: if the module containing the package supports graph + // pruning then it should satisfy the import invariant itself, so all of + // its dependencies should be in its go.mod file, and if the module + // containing the package does not support pruning then if we make it a + // root we will load all of its (unpruned) transitive dependencies into + // the module graph. // - // (This is the “argument invariant” of lazy loading, and is important for + // (This is the “argument invariant”, and is important for // reproducibility.) default: @@ -807,16 +922,15 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen // We've added or upgraded one or more roots, so load the full module // graph so that we can update those roots to be consistent with other // requirements. - if cfg.BuildMod != "mod" { + if mustHaveCompleteRequirements() { // Our changes to the roots may have moved dependencies into or out of - // the lazy-loading horizon, which could in turn change the selected - // versions of other modules. (Unlike for eager modules, for lazy - // modules adding or removing an explicit root is a semantic change, not - // just a cosmetic one.) + // the graph-pruning horizon, which could in turn change the selected + // versions of other modules. (For pruned modules adding or removing an + // explicit root is a semantic change, not just a cosmetic one.) return rs, errGoModDirty } - rs = newRequirements(lazy, roots, direct) + rs = newRequirements(pruned, roots, direct) var err error mg, err = rs.Graph(ctx) if err != nil { @@ -831,7 +945,7 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen if rs.graph.Load() != nil { // We've already loaded the full module graph, which includes the // requirements of all of the root modules — even the transitive - // requirements, if they are eager! + // requirements, if they are unpruned! mg, _ = rs.Graph(ctx) } else if cfg.BuildMod == "vendor" { // We can't spot-check the requirements of other modules because we @@ -855,7 +969,9 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen roots = make([]module.Version, 0, len(rs.rootModules)) rootsUpgraded = false inRootPaths := make(map[string]bool, len(rs.rootModules)+1) - inRootPaths[Target.Path] = true + for _, mm := range MainModules.Versions() { + inRootPaths[mm.Path] = true + } for _, m := range rs.rootModules { if inRootPaths[m.Path] { // This root specifies a redundant path. We already retained the @@ -908,12 +1024,12 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen } } - if rs.depth == lazy && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { - // The root set is unchanged and rs was already lazy, so keep rs to + if rs.pruning == pruned && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { + // The root set is unchanged and rs was already pruned, so keep rs to // preserve its cached ModuleGraph (if any). return rs, nil } - return newRequirements(lazy, roots, direct), nil + return newRequirements(pruned, roots, direct), nil } // spotCheckRoots reports whether the versions of the roots in rs satisfy the @@ -955,17 +1071,37 @@ func spotCheckRoots(ctx context.Context, rs *Requirements, mods map[module.Versi return true } -// tidyEagerRoots returns a minimal set of root requirements that maintains the -// selected version of every module that provided a package in pkgs, and -// includes the selected version of every such module in direct as a root. -func tidyEagerRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { +// tidyUnprunedRoots returns a minimal set of root requirements that maintains +// the selected version of every module that provided or lexically could have +// provided a package in pkgs, and includes the selected version of every such +// module in direct as a root. +func tidyUnprunedRoots(ctx context.Context, mainModule module.Version, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { var ( + // keep is a set of of modules that provide packages or are needed to + // disambiguate imports. keep []module.Version keptPath = map[string]bool{} - ) - var ( - rootPaths []string // module paths that should be included as roots + + // rootPaths is a list of module paths that provide packages directly + // imported from the main module. They should be included as roots. + rootPaths []string inRootPaths = map[string]bool{} + + // altMods is a set of paths of modules that lexically could have provided + // imported packages. It may be okay to remove these from the list of + // explicit requirements if that removes them from the module graph. If they + // are present in the module graph reachable from rootPaths, they must not + // be at a lower version. That could cause a missing sum error or a new + // import ambiguity. + // + // For example, suppose a developer rewrites imports from example.com/m to + // example.com/m/v2, then runs 'go mod tidy'. Tidy may delete the + // requirement on example.com/m if there is no other transitive requirement + // on it. However, if example.com/m were downgraded to a version not in + // go.sum, when package example.com/m/v2/p is loaded, we'd get an error + // trying to disambiguate the import, since we can't check example.com/m + // without its sum. See #47738. + altMods = map[string]string{} ) for _, pkg := range pkgs { if !pkg.fromExternalModule() { @@ -979,16 +1115,48 @@ func tidyEagerRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg inRootPaths[m.Path] = true } } + for _, m := range pkg.altMods { + altMods[m.Path] = m.Version + } } - min, err := mvs.Req(Target, rootPaths, &mvsReqs{roots: keep}) + // Construct a build list with a minimal set of roots. + // This may remove or downgrade modules in altMods. + reqs := &mvsReqs{roots: keep} + min, err := mvs.Req(mainModule, rootPaths, reqs) if err != nil { return nil, err } - return newRequirements(eager, min, direct), nil + buildList, err := mvs.BuildList([]module.Version{mainModule}, reqs) + if err != nil { + return nil, err + } + + // Check if modules in altMods were downgraded but not removed. + // If so, add them to roots, which will retain an "// indirect" requirement + // in go.mod. See comment on altMods above. + keptAltMod := false + for _, m := range buildList { + if v, ok := altMods[m.Path]; ok && semver.Compare(m.Version, v) < 0 { + keep = append(keep, module.Version{Path: m.Path, Version: v}) + keptAltMod = true + } + } + if keptAltMod { + // We must run mvs.Req again instead of simply adding altMods to min. + // It's possible that a requirement in altMods makes some other + // explicit indirect requirement unnecessary. + reqs.roots = keep + min, err = mvs.Req(mainModule, rootPaths, reqs) + if err != nil { + return nil, err + } + } + + return newRequirements(unpruned, min, direct), nil } -// updateEagerRoots returns a set of root requirements that includes the selected +// updateUnprunedRoots returns a set of root requirements that includes the selected // version of every module path in direct as a root, and maintains the selected // version of every module selected in the graph of rs. // @@ -1002,7 +1170,7 @@ func tidyEagerRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg // by a dependency in add. // 4. Every version in add is selected at its given version unless upgraded by // (the dependencies of) an existing root or another module in add. -func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requirements, add []module.Version) (*Requirements, error) { +func updateUnprunedRoots(ctx context.Context, direct map[string]bool, rs *Requirements, add []module.Version) (*Requirements, error) { mg, err := rs.Graph(ctx) if err != nil { // We can't ignore errors in the module graph even if the user passed the -e @@ -1011,7 +1179,7 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme return rs, err } - if cfg.BuildMod != "mod" { + if mustHaveCompleteRequirements() { // Instead of actually updating the requirements, just check that no updates // are needed. if rs == nil { @@ -1067,10 +1235,10 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme // “The selected version of every module path in direct is included as a root.” // - // This is only for convenience and clarity for end users: in an eager module, + // This is only for convenience and clarity for end users: in an unpruned module, // the choice of explicit vs. implicit dependency has no impact on MVS // selection (for itself or any other module). - keep := append(mg.BuildList()[1:], add...) + keep := append(mg.BuildList()[MainModules.Len():], add...) for _, m := range keep { if direct[m.Path] && !inRootPaths[m.Path] { rootPaths = append(rootPaths, m.Path) @@ -1078,44 +1246,53 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme } } - min, err := mvs.Req(Target, rootPaths, &mvsReqs{roots: keep}) - if err != nil { - return rs, err + var roots []module.Version + for _, mainModule := range MainModules.Versions() { + min, err := mvs.Req(mainModule, rootPaths, &mvsReqs{roots: keep}) + if err != nil { + return rs, err + } + roots = append(roots, min...) } - if rs.depth == eager && reflect.DeepEqual(min, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { - // The root set is unchanged and rs was already eager, so keep rs to + if MainModules.Len() > 1 { + module.Sort(roots) + } + if rs.pruning == unpruned && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { + // The root set is unchanged and rs was already unpruned, so keep rs to // preserve its cached ModuleGraph (if any). return rs, nil } - return newRequirements(eager, min, direct), nil + + return newRequirements(unpruned, roots, direct), nil } -// convertDepth returns a version of rs with the given depth. -// If rs already has the given depth, convertDepth returns rs unmodified. -func convertDepth(ctx context.Context, rs *Requirements, depth modDepth) (*Requirements, error) { - if rs.depth == depth { +// convertPruning returns a version of rs with the given pruning behavior. +// If rs already has the given pruning, convertPruning returns rs unmodified. +func convertPruning(ctx context.Context, rs *Requirements, pruning modPruning) (*Requirements, error) { + if rs.pruning == pruning { return rs, nil + } else if rs.pruning == workspace || pruning == workspace { + panic("attempthing to convert to/from workspace pruning and another pruning type") } - if depth == eager { - // We are converting a lazy module to an eager one. The roots of an eager - // module graph are a superset of the roots of a lazy graph, so we don't - // need to add any new roots — we just need to prune away the ones that are - // redundant given eager loading, which is exactly what updateEagerRoots - // does. - return updateEagerRoots(ctx, rs.direct, rs, nil) + if pruning == unpruned { + // We are converting a pruned module to an unpruned one. The roots of a + // ppruned module graph are a superset of the roots of an unpruned one, so + // we don't need to add any new roots — we just need to drop the ones that + // are redundant, which is exactly what updateUnprunedRoots does. + return updateUnprunedRoots(ctx, rs.direct, rs, nil) } - // We are converting an eager module to a lazy one. The module graph of an - // eager module includes the transitive dependencies of every module in the - // build list. + // We are converting an unpruned module to a pruned one. // - // Hey, we can express that as a lazy root set! “Include the transitive - // dependencies of every module in the build list” is exactly what happens in - // a lazy module if we promote every module in the build list to a root! + // An unpruned module graph includes the transitive dependencies of every + // module in the build list. As it turns out, we can express that as a pruned + // root set! “Include the transitive dependencies of every module in the build + // list” is exactly what happens in a pruned module if we promote every module + // in the build list to a root. mg, err := rs.Graph(ctx) if err != nil { return rs, err } - return newRequirements(lazy, mg.BuildList()[1:], rs.direct), nil + return newRequirements(pruned, mg.BuildList()[MainModules.Len():], rs.direct), nil } diff --git a/libgo/go/cmd/go/internal/modload/edit.go b/libgo/go/cmd/go/internal/modload/edit.go index 47f236ce168..0f37e3b2e94 100644 --- a/libgo/go/cmd/go/internal/modload/edit.go +++ b/libgo/go/cmd/go/internal/modload/edit.go @@ -21,7 +21,7 @@ import ( // 2. Each module version in tryUpgrade is upgraded toward the indicated // version as far as can be done without violating (1). // -// 3. Each module version in rs.rootModules (or rs.graph, if rs.depth is eager) +// 3. Each module version in rs.rootModules (or rs.graph, if rs is unpruned) // is downgraded from its original version only to the extent needed to // satisfy (1), or upgraded only to the extent needed to satisfy (1) and // (2). @@ -69,13 +69,14 @@ func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSel } var roots []module.Version - if rs.depth == eager { - // In an eager module, modules that provide packages imported by the main - // module may either be explicit roots or implicit transitive dependencies. - // We promote the modules in mustSelect to be explicit requirements. + if rs.pruning == unpruned { + // In a module without graph pruning, modules that provide packages imported + // by the main module may either be explicit roots or implicit transitive + // dependencies. We promote the modules in mustSelect to be explicit + // requirements. var rootPaths []string for _, m := range mustSelect { - if m.Version != "none" && m.Path != Target.Path { + if m.Version != "none" && !MainModules.Contains(m.Path) { rootPaths = append(rootPaths, m.Path) } } @@ -97,13 +98,13 @@ func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSel } } - roots, err = mvs.Req(Target, rootPaths, &mvsReqs{roots: mods}) + roots, err = mvs.Req(MainModules.mustGetSingleMainModule(), rootPaths, &mvsReqs{roots: mods}) if err != nil { return nil, false, err } } else { - // In a lazy module, every module that provides a package imported by the - // main module must be retained as a root. + // In a module with a pruned graph, every module that provides a package + // imported by the main module must be retained as a root. roots = mods if !changed { // Because the roots we just computed are unchanged, the entire graph must @@ -126,7 +127,7 @@ func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSel direct[m.Path] = true } } - return newRequirements(rs.depth, roots, direct), changed, nil + return newRequirements(rs.pruning, roots, direct), changed, nil } // limiterForEdit returns a versionLimiter with its max versions set such that @@ -149,11 +150,12 @@ func limiterForEdit(ctx context.Context, rs *Requirements, tryUpgrade, mustSelec } } - if rs.depth == eager { - // Eager go.mod files don't indicate which transitive dependencies are - // actually relevant to the main module, so we have to assume that any module - // that could have provided any package — that is, any module whose selected - // version was not "none" — may be relevant. + if rs.pruning == unpruned { + // go.mod files that do not support graph pruning don't indicate which + // transitive dependencies are actually relevant to the main module, so we + // have to assume that any module that could have provided any package — + // that is, any module whose selected version was not "none" — may be + // relevant. for _, m := range mg.BuildList() { restrictTo(m) } @@ -175,7 +177,7 @@ func limiterForEdit(ctx context.Context, rs *Requirements, tryUpgrade, mustSelec } } - if err := raiseLimitsForUpgrades(ctx, maxVersion, rs.depth, tryUpgrade, mustSelect); err != nil { + if err := raiseLimitsForUpgrades(ctx, maxVersion, rs.pruning, tryUpgrade, mustSelect); err != nil { return nil, err } @@ -185,7 +187,7 @@ func limiterForEdit(ctx context.Context, rs *Requirements, tryUpgrade, mustSelec restrictTo(m) } - return newVersionLimiter(rs.depth, maxVersion), nil + return newVersionLimiter(rs.pruning, maxVersion), nil } // raiseLimitsForUpgrades increases the module versions in maxVersions to the @@ -195,12 +197,12 @@ func limiterForEdit(ctx context.Context, rs *Requirements, tryUpgrade, mustSelec // // Versions not present in maxVersion are unrestricted, and it is assumed that // they will not be promoted to root requirements (and thus will not contribute -// their own dependencies if the main module is lazy). +// their own dependencies if the main module supports graph pruning). // // These limits provide an upper bound on how far a module may be upgraded as // part of an incidental downgrade, if downgrades are needed in order to select // the versions in mustSelect. -func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, depth modDepth, tryUpgrade []module.Version, mustSelect []module.Version) error { +func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, pruning modPruning, tryUpgrade []module.Version, mustSelect []module.Version) error { // allow raises the limit for m.Path to at least m.Version. // If m.Path was already unrestricted, it remains unrestricted. allow := func(m module.Version) { @@ -214,21 +216,21 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d } var ( - eagerUpgrades []module.Version - isLazyRootPath map[string]bool + unprunedUpgrades []module.Version + isPrunedRootPath map[string]bool ) - if depth == eager { - eagerUpgrades = tryUpgrade + if pruning == unpruned { + unprunedUpgrades = tryUpgrade } else { - isLazyRootPath = make(map[string]bool, len(maxVersion)) + isPrunedRootPath = make(map[string]bool, len(maxVersion)) for p := range maxVersion { - isLazyRootPath[p] = true + isPrunedRootPath[p] = true } for _, m := range tryUpgrade { - isLazyRootPath[m.Path] = true + isPrunedRootPath[m.Path] = true } for _, m := range mustSelect { - isLazyRootPath[m.Path] = true + isPrunedRootPath[m.Path] = true } allowedRoot := map[module.Version]bool{} @@ -240,10 +242,10 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d } allowedRoot[m] = true - if m.Path == Target.Path { - // Target is already considered to be higher than any possible m, so we - // won't be upgrading to it anyway and there is no point scanning its - // dependencies. + if MainModules.Contains(m.Path) { + // The main module versions are already considered to be higher than any + // possible m, so m cannot be selected as a root and there is no point + // scanning its dependencies. return nil } @@ -253,15 +255,15 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d if err != nil { return err } - if summary.depth == eager { - // For efficiency, we'll load all of the eager upgrades as one big + if summary.pruning == unpruned { + // For efficiency, we'll load all of the unpruned upgrades as one big // graph, rather than loading the (potentially-overlapping) subgraph for // each upgrade individually. - eagerUpgrades = append(eagerUpgrades, m) + unprunedUpgrades = append(unprunedUpgrades, m) return nil } for _, r := range summary.require { - if isLazyRootPath[r.Path] { + if isPrunedRootPath[r.Path] { // r could become a root as the result of an upgrade or downgrade, // in which case its dependencies will not be pruned out. // We need to allow those dependencies to be upgraded too. @@ -282,21 +284,19 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d } } - if len(eagerUpgrades) > 0 { - // Compute the max versions for eager upgrades all together. - // Since these modules are eager, we'll end up scanning all of their + if len(unprunedUpgrades) > 0 { + // Compute the max versions for unpruned upgrades all together. + // Since these modules are unpruned, we'll end up scanning all of their // transitive dependencies no matter which versions end up selected, // and since we have a large dependency graph to scan we might get // a significant benefit from not revisiting dependencies that are at // common versions among multiple upgrades. - upgradeGraph, err := readModGraph(ctx, eager, eagerUpgrades) + upgradeGraph, err := readModGraph(ctx, unpruned, unprunedUpgrades) if err != nil { - if go117LazyTODO { - // Compute the requirement path from a module path in tryUpgrade to the - // error, and the requirement path (if any) from rs.rootModules to the - // tryUpgrade module path. Return a *mvs.BuildListError showing the - // concatenation of the paths (with an upgrade in the middle). - } + // Compute the requirement path from a module path in tryUpgrade to the + // error, and the requirement path (if any) from rs.rootModules to the + // tryUpgrade module path. Return a *mvs.BuildListError showing the + // concatenation of the paths (with an upgrade in the middle). return err } @@ -311,7 +311,7 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d nextRoots := append([]module.Version(nil), mustSelect...) for nextRoots != nil { module.Sort(nextRoots) - rs := newRequirements(depth, nextRoots, nil) + rs := newRequirements(pruning, nextRoots, nil) nextRoots = nil rs, mustGraph, err := expandGraph(ctx, rs) @@ -325,7 +325,7 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d // which case we will error out either way). allow(r) - if isLazyRootPath[r.Path] { + if isPrunedRootPath[r.Path] { if v, ok := rs.rootSelected(r.Path); ok && r.Version == v { // r is already a root, so its requirements are already included in // the build list. @@ -365,12 +365,12 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit } var initial []module.Version - if rs.depth == eager { + if rs.pruning == unpruned { mg, err := rs.Graph(ctx) if err != nil { return nil, false, err } - initial = mg.BuildList()[1:] + initial = mg.BuildList()[MainModules.Len():] } else { initial = rs.rootModules } @@ -382,7 +382,7 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit mods = make([]module.Version, 0, len(limiter.selected)) for path, v := range limiter.selected { - if v != "none" && path != Target.Path { + if v != "none" && !MainModules.Contains(path) { mods = append(mods, module.Version{Path: path, Version: v}) } } @@ -392,13 +392,13 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit // downgraded module may require a higher (but still allowed) version of // another. The lower version may require extraneous dependencies that aren't // actually relevant, so we need to compute the actual selected versions. - mg, err := readModGraph(ctx, rs.depth, mods) + mg, err := readModGraph(ctx, rs.pruning, mods) if err != nil { return nil, false, err } mods = make([]module.Version, 0, len(limiter.selected)) for path, _ := range limiter.selected { - if path != Target.Path { + if !MainModules.Contains(path) { if v := mg.Selected(path); v != "none" { mods = append(mods, module.Version{Path: path, Version: v}) } @@ -414,16 +414,16 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit // A versionLimiter tracks the versions that may be selected for each module // subject to constraints on the maximum versions of transitive dependencies. type versionLimiter struct { - // depth is the depth at which the dependencies of the modules passed to + // pruning is the pruning at which the dependencies of the modules passed to // Select and UpgradeToward are loaded. - depth modDepth + pruning modPruning // max maps each module path to the maximum version that may be selected for // that path. // // Paths with no entry are unrestricted, and we assume that they will not be // promoted to root dependencies (so will not contribute dependencies if the - // main module is lazy). + // main module supports graph pruning). max map[string]string // selected maps each module path to a version of that path (if known) whose @@ -475,14 +475,18 @@ func (dq dqState) isDisqualified() bool { // in the map are unrestricted. The limiter assumes that unrestricted paths will // not be promoted to root dependencies. // -// If depth is lazy, then if a module passed to UpgradeToward or Select is -// itself lazy, its unrestricted dependencies are skipped when scanning -// requirements. -func newVersionLimiter(depth modDepth, max map[string]string) *versionLimiter { +// If module graph pruning is in effect, then if a module passed to +// UpgradeToward or Select supports pruning, its unrestricted dependencies are +// skipped when scanning requirements. +func newVersionLimiter(pruning modPruning, max map[string]string) *versionLimiter { + selected := make(map[string]string) + for _, m := range MainModules.Versions() { + selected[m.Path] = m.Version + } return &versionLimiter{ - depth: depth, + pruning: pruning, max: max, - selected: map[string]string{Target.Path: Target.Version}, + selected: selected, dqReason: map[module.Version]dqState{}, requiring: map[module.Version][]module.Version{}, } @@ -491,8 +495,8 @@ func newVersionLimiter(depth modDepth, max map[string]string) *versionLimiter { // UpgradeToward attempts to upgrade the selected version of m.Path as close as // possible to m.Version without violating l's maximum version limits. // -// If depth is lazy and m itself is lazy, the the dependencies of unrestricted -// dependencies of m will not be followed. +// If module graph pruning is in effect and m itself supports pruning, the +// dependencies of unrestricted dependencies of m will not be followed. func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) error { selected, ok := l.selected[m.Path] if ok { @@ -504,7 +508,7 @@ func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) er selected = "none" } - if l.check(m, l.depth).isDisqualified() { + if l.check(m, l.pruning).isDisqualified() { candidates, err := versions(ctx, m.Path, CheckAllowed) if err != nil { // This is likely a transient error reaching the repository, @@ -521,7 +525,7 @@ func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) er }) candidates = candidates[:i] - for l.check(m, l.depth).isDisqualified() { + for l.check(m, l.pruning).isDisqualified() { n := len(candidates) if n == 0 || cmpVersion(selected, candidates[n-1]) >= 0 { // We couldn't find a suitable candidate above the already-selected version. @@ -538,7 +542,7 @@ func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) er // Select attempts to set the selected version of m.Path to exactly m.Version. func (l *versionLimiter) Select(m module.Version) (conflict module.Version, err error) { - dq := l.check(m, l.depth) + dq := l.check(m, l.pruning) if !dq.isDisqualified() { l.selected[m.Path] = m.Version } @@ -548,15 +552,15 @@ func (l *versionLimiter) Select(m module.Version) (conflict module.Version, err // check determines whether m (or its transitive dependencies) would violate l's // maximum version limits if added to the module requirement graph. // -// If depth is lazy and m itself is lazy, then the dependencies of unrestricted -// dependencies of m will not be followed. If the lazy loading invariants hold -// for the main module up to this point, the packages in those modules are at -// best only imported by tests of dependencies that are themselves loaded from -// outside modules. Although we would like to keep 'go test all' as reproducible -// as is feasible, we don't want to retain test dependencies that are only -// marginally relevant at best. -func (l *versionLimiter) check(m module.Version, depth modDepth) dqState { - if m.Version == "none" || m == Target { +// If pruning is in effect and m itself supports graph pruning, the dependencies +// of unrestricted dependencies of m will not be followed. If the graph-pruning +// invariants hold for the main module up to this point, the packages in those +// modules are at best only imported by tests of dependencies that are +// themselves loaded from outside modules. Although we would like to keep +// 'go test all' as reproducible as is feasible, we don't want to retain test +// dependencies that are only marginally relevant at best. +func (l *versionLimiter) check(m module.Version, pruning modPruning) dqState { + if m.Version == "none" || m == MainModules.mustGetSingleMainModule() { // version "none" has no requirements, and the dependencies of Target are // tautological. return dqState{} @@ -586,20 +590,20 @@ func (l *versionLimiter) check(m module.Version, depth modDepth) dqState { return l.disqualify(m, dqState{err: err}) } - if summary.depth == eager { - depth = eager + if summary.pruning == unpruned { + pruning = unpruned } for _, r := range summary.require { - if depth == lazy { + if pruning == pruned { if _, restricted := l.max[r.Path]; !restricted { // r.Path is unrestricted, so we don't care at what version it is // selected. We assume that r.Path will not become a root dependency, so - // since m is lazy, r's dependencies won't be followed. + // since m supports pruning, r's dependencies won't be followed. continue } } - if dq := l.check(r, depth); dq.isDisqualified() { + if dq := l.check(r, pruning); dq.isDisqualified() { return l.disqualify(m, dq) } diff --git a/libgo/go/cmd/go/internal/modload/import.go b/libgo/go/cmd/go/internal/modload/import.go index d2bbe5cbe0b..812e48a1568 100644 --- a/libgo/go/cmd/go/internal/modload/import.go +++ b/libgo/go/cmd/go/internal/modload/import.go @@ -32,6 +32,8 @@ type ImportMissingError struct { Module module.Version QueryErr error + ImportingMainModule module.Version + // isStd indicates whether we would expect to find the package in the standard // library. This is normally true for all dotless import paths, but replace // directives can cause us to treat the replaced paths as also being in @@ -71,6 +73,9 @@ func (e *ImportMissingError) Error() string { if e.QueryErr != nil { return fmt.Sprintf("%s: %v", message, e.QueryErr) } + if e.ImportingMainModule.Path != "" && e.ImportingMainModule != MainModules.ModContainingCWD() { + return fmt.Sprintf("%s; to add it:\n\tcd %s\n\tgo get %s", message, MainModules.ModRoot(e.ImportingMainModule), e.Path) + } return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path) } @@ -238,55 +243,63 @@ func (e *invalidImportError) Unwrap() error { // // If the package is not present in any module selected from the requirement // graph, importFromModules returns an *ImportMissingError. -func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph) (m module.Version, dir string, err error) { +// +// If the package is present in exactly one module, importFromModules will +// return the module, its root directory, and a list of other modules that +// lexically could have provided the package but did not. +func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph) (m module.Version, dir string, altMods []module.Version, err error) { if strings.Contains(path, "@") { - return module.Version{}, "", fmt.Errorf("import path should not have @version") + return module.Version{}, "", nil, fmt.Errorf("import path should not have @version") } if build.IsLocalImport(path) { - return module.Version{}, "", fmt.Errorf("relative import not supported") + return module.Version{}, "", nil, fmt.Errorf("relative import not supported") } if path == "C" { // There's no directory for import "C". - return module.Version{}, "", nil + return module.Version{}, "", nil, nil } // Before any further lookup, check that the path is valid. if err := module.CheckImportPath(path); err != nil { - return module.Version{}, "", &invalidImportError{importPath: path, err: err} + return module.Version{}, "", nil, &invalidImportError{importPath: path, err: err} } // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) if pathIsStd && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { - if targetInGorootSrc { - if dir, ok, err := dirInModule(path, targetPrefix, ModRoot(), true); err != nil { - return module.Version{}, dir, err - } else if ok { - return Target, dir, nil + for _, mainModule := range MainModules.Versions() { + if MainModules.InGorootSrc(mainModule) { + if dir, ok, err := dirInModule(path, MainModules.PathPrefix(mainModule), MainModules.ModRoot(mainModule), true); err != nil { + return module.Version{}, dir, nil, err + } else if ok { + return mainModule, dir, nil, nil + } } } dir := filepath.Join(cfg.GOROOT, "src", path) - return module.Version{}, dir, nil + return module.Version{}, dir, nil, nil } // -mod=vendor is special. // Everything must be in the main module or the main module's vendor directory. if cfg.BuildMod == "vendor" { - mainDir, mainOK, mainErr := dirInModule(path, targetPrefix, ModRoot(), true) - vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(ModRoot(), "vendor"), false) + mainModule := MainModules.mustGetSingleMainModule() + modRoot := MainModules.ModRoot(mainModule) + mainDir, mainOK, mainErr := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true) + vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(modRoot, "vendor"), false) if mainOK && vendorOK { - return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} + return module.Version{}, "", nil, &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} } // Prefer to return main directory if there is one, // Note that we're not checking that the package exists. // We'll leave that for load. if !vendorOK && mainDir != "" { - return Target, mainDir, nil + return mainModule, mainDir, nil, nil } if mainErr != nil { - return module.Version{}, "", mainErr + return module.Version{}, "", nil, mainErr } - readVendorList() - return vendorPkgModule[path], vendorDir, nil + readVendorList(mainModule) + return vendorPkgModule[path], vendorDir, nil, nil } // Check each module on the build list. @@ -307,7 +320,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // already non-nil, then we attempt to load the package using the full // requirements in mg. for { - var sumErrMods []module.Version + var sumErrMods, altMods []module.Version for prefix := path; prefix != "."; prefix = pathpkg.Dir(prefix) { var ( v string @@ -341,13 +354,15 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // continue the loop and find the package in some other module, // we need to look at this module to make sure the import is // not ambiguous. - return module.Version{}, "", err + return module.Version{}, "", nil, err } if dir, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return module.Version{}, "", err + return module.Version{}, "", nil, err } else if ok { mods = append(mods, m) dirs = append(dirs, dir) + } else { + altMods = append(altMods, m) } } @@ -360,7 +375,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M mods[i], mods[j] = mods[j], mods[i] dirs[i], dirs[j] = dirs[j], dirs[i] } - return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + return module.Version{}, "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } if len(sumErrMods) > 0 { @@ -368,7 +383,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M j := len(sumErrMods) - 1 - i sumErrMods[i], sumErrMods[j] = sumErrMods[j], sumErrMods[i] } - return module.Version{}, "", &ImportMissingSumError{ + return module.Version{}, "", nil, &ImportMissingSumError{ importPath: path, mods: sumErrMods, found: len(mods) > 0, @@ -376,7 +391,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M } if len(mods) == 1 { - return mods[0], dirs[0], nil + return mods[0], dirs[0], altMods, nil } if mg != nil { @@ -386,7 +401,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M if !HasModRoot() { queryErr = ErrNoModRoot } - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} + return module.Version{}, "", nil, &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} } // So far we've checked the root dependencies. @@ -397,7 +412,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // the module graph, so we can't return an ImportMissingError here — one // of the missing modules might actually contain the package in question, // in which case we shouldn't go looking for it in some new dependency. - return module.Version{}, "", err + return module.Version{}, "", nil, err } } } @@ -410,9 +425,9 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M func queryImport(ctx context.Context, path string, rs *Requirements) (module.Version, error) { // To avoid spurious remote fetches, try the latest replacement for each // module (golang.org/issue/26241). - if index != nil { - var mods []module.Version - for mp, mv := range index.highestReplaced { + var mods []module.Version + if MainModules != nil { // TODO(#48912): Ensure MainModules exists at this point, and remove the check. + for mp, mv := range MainModules.HighestReplaced() { if !maybeInModule(path, mp) { continue } @@ -439,40 +454,41 @@ func queryImport(ctx context.Context, path string, rs *Requirements) (module.Ver } mods = append(mods, module.Version{Path: mp, Version: mv}) } + } - // Every module path in mods is a prefix of the import path. - // As in QueryPattern, prefer the longest prefix that satisfies the import. - sort.Slice(mods, func(i, j int) bool { - return len(mods[i].Path) > len(mods[j].Path) - }) - for _, m := range mods { - needSum := true - root, isLocal, err := fetch(ctx, m, needSum) - if err != nil { - if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { - return module.Version{}, &ImportMissingSumError{importPath: path} - } - return module.Version{}, err - } - if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, err - } else if ok { - if cfg.BuildMod == "readonly" { - return module.Version{}, &ImportMissingError{Path: path, replaced: m} - } - return m, nil + // Every module path in mods is a prefix of the import path. + // As in QueryPattern, prefer the longest prefix that satisfies the import. + sort.Slice(mods, func(i, j int) bool { + return len(mods[i].Path) > len(mods[j].Path) + }) + for _, m := range mods { + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} } + return module.Version{}, err } - if len(mods) > 0 && module.CheckPath(path) != nil { - // The package path is not valid to fetch remotely, - // so it can only exist in a replaced module, - // and we know from the above loop that it is not. - return module.Version{}, &PackageNotInModuleError{ - Mod: mods[0], - Query: "latest", - Pattern: path, - Replacement: Replacement(mods[0]), + if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return m, err + } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} } + return m, nil + } + } + if len(mods) > 0 && module.CheckPath(path) != nil { + // The package path is not valid to fetch remotely, + // so it can only exist in a replaced module, + // and we know from the above loop that it is not. + replacement := Replacement(mods[0]) + return module.Version{}, &PackageNotInModuleError{ + Mod: mods[0], + Query: "latest", + Pattern: path, + Replacement: replacement, } } @@ -596,7 +612,7 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // (the main module, and any directory trees pointed at by replace directives). if isLocal { for d := dir; d != mdir && len(d) > len(mdir); { - haveGoMod := haveGoModCache.Do(d, func() interface{} { + haveGoMod := haveGoModCache.Do(d, func() any { fi, err := fsys.Stat(filepath.Join(d, "go.mod")) return err == nil && !fi.IsDir() }).(bool) @@ -619,7 +635,7 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // Are there Go source files in the directory? // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. - res := haveGoFilesCache.Do(dir, func() interface{} { + res := haveGoFilesCache.Do(dir, func() any { ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) @@ -638,14 +654,14 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + return modRoot, true, nil } if r := Replacement(mod); r.Path != "" { if r.Version == "" { dir = r.Path if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) + dir = filepath.Join(replaceRelativeTo(), dir) } // Ensure that the replacement directory actually exists: // dirInModule does not report errors for missing modules, @@ -667,7 +683,7 @@ func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, i mod = r } - if HasModRoot() && cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + if HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() && needSum && !modfetch.HaveSum(mod) { return "", false, module.VersionError(mod, &sumMissingError{}) } diff --git a/libgo/go/cmd/go/internal/modload/import_test.go b/libgo/go/cmd/go/internal/modload/import_test.go index 98145887e9d..65a889ec52f 100644 --- a/libgo/go/cmd/go/internal/modload/import_test.go +++ b/libgo/go/cmd/go/internal/modload/import_test.go @@ -69,7 +69,7 @@ func TestQueryImport(t *testing.T) { RootMode = NoRoot ctx := context.Background() - rs := newRequirements(eager, nil, nil) + rs := LoadModFile(ctx) for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { diff --git a/libgo/go/cmd/go/internal/modload/init.go b/libgo/go/cmd/go/internal/modload/init.go index 45f724d5e36..cdcfbeb8ded 100644 --- a/libgo/go/cmd/go/internal/modload/init.go +++ b/libgo/go/cmd/go/internal/modload/init.go @@ -12,11 +12,13 @@ import ( "fmt" "go/build" "internal/lazyregexp" + "io/ioutil" "os" "path" "path/filepath" "strconv" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -43,28 +45,198 @@ var ( ForceUseModules bool allowMissingModuleImports bool + + // ExplicitWriteGoMod prevents LoadPackages, ListModules, and other functions + // from updating go.mod and go.sum or reporting errors when updates are + // needed. A package should set this if it would cause go.mod to be written + // multiple times (for example, 'go get' calls LoadPackages multiple times) or + // if it needs some other operation to be successful before go.mod and go.sum + // can be written (for example, 'go mod download' must download modules before + // adding sums to go.sum). Packages that set this are responsible for calling + // WriteGoMod explicitly. + ExplicitWriteGoMod bool ) // Variables set in Init. var ( initialized bool - modRoot string - gopath string + + // These are primarily used to initialize the MainModules, and should be + // eventually superceded by them but are still used in cases where the module + // roots are required but MainModules hasn't been initialized yet. Set to + // the modRoots of the main modules. + // modRoots != nil implies len(modRoots) > 0 + modRoots []string + gopath string ) -// Variables set in initTarget (during {Load,Create}ModFile). +// EnterModule resets MainModules and requirements to refer to just this one module. +func EnterModule(ctx context.Context, enterModroot string) { + MainModules = nil // reset MainModules + requirements = nil + workFilePath = "" // Force module mode + modfetch.Reset() + + modRoots = []string{enterModroot} + LoadModFile(ctx) +} + +// Variable set in InitWorkfile var ( - Target module.Version + // Set to the path to the go.work file, or "" if workspace mode is disabled. + workFilePath string +) + +type MainModuleSet struct { + // versions are the module.Version values of each of the main modules. + // For each of them, the Path fields are ordinary module paths and the Version + // fields are empty strings. + versions []module.Version + + // modRoot maps each module in versions to its absolute filesystem path. + modRoot map[module.Version]string - // targetPrefix is the path prefix for packages in Target, without a trailing - // slash. For most modules, targetPrefix is just Target.Path, but the + // pathPrefix is the path prefix for packages in the module, without a trailing + // slash. For most modules, pathPrefix is just version.Path, but the // standard-library module "std" has an empty prefix. - targetPrefix string + pathPrefix map[module.Version]string - // targetInGorootSrc caches whether modRoot is within GOROOT/src. + // inGorootSrc caches whether modRoot is within GOROOT/src. // The "std" module is special within GOROOT/src, but not otherwise. - targetInGorootSrc bool -) + inGorootSrc map[module.Version]bool + + modFiles map[module.Version]*modfile.File + + modContainingCWD module.Version + + workFileGoVersion string + + workFileReplaceMap map[module.Version]module.Version + // highest replaced version of each module path; empty string for wildcard-only replacements + highestReplaced map[string]string + + indexMu sync.Mutex + indices map[module.Version]*modFileIndex +} + +func (mms *MainModuleSet) PathPrefix(m module.Version) string { + return mms.pathPrefix[m] +} + +// Versions returns the module.Version values of each of the main modules. +// For each of them, the Path fields are ordinary module paths and the Version +// fields are empty strings. +// Callers should not modify the returned slice. +func (mms *MainModuleSet) Versions() []module.Version { + if mms == nil { + return nil + } + return mms.versions +} + +func (mms *MainModuleSet) Contains(path string) bool { + if mms == nil { + return false + } + for _, v := range mms.versions { + if v.Path == path { + return true + } + } + return false +} + +func (mms *MainModuleSet) ModRoot(m module.Version) string { + if mms == nil { + return "" + } + return mms.modRoot[m] +} + +func (mms *MainModuleSet) InGorootSrc(m module.Version) bool { + if mms == nil { + return false + } + return mms.inGorootSrc[m] +} + +func (mms *MainModuleSet) mustGetSingleMainModule() module.Version { + if mms == nil || len(mms.versions) == 0 { + panic("internal error: mustGetSingleMainModule called in context with no main modules") + } + if len(mms.versions) != 1 { + if inWorkspaceMode() { + panic("internal error: mustGetSingleMainModule called in workspace mode") + } else { + panic("internal error: multiple main modules present outside of workspace mode") + } + } + return mms.versions[0] +} + +func (mms *MainModuleSet) GetSingleIndexOrNil() *modFileIndex { + if mms == nil { + return nil + } + if len(mms.versions) == 0 { + return nil + } + return mms.indices[mms.mustGetSingleMainModule()] +} + +func (mms *MainModuleSet) Index(m module.Version) *modFileIndex { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + return mms.indices[m] +} + +func (mms *MainModuleSet) SetIndex(m module.Version, index *modFileIndex) { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + mms.indices[m] = index +} + +func (mms *MainModuleSet) ModFile(m module.Version) *modfile.File { + return mms.modFiles[m] +} + +func (mms *MainModuleSet) Len() int { + if mms == nil { + return 0 + } + return len(mms.versions) +} + +// ModContainingCWD returns the main module containing the working directory, +// or module.Version{} if none of the main modules contain the working +// directory. +func (mms *MainModuleSet) ModContainingCWD() module.Version { + return mms.modContainingCWD +} + +func (mms *MainModuleSet) HighestReplaced() map[string]string { + return mms.highestReplaced +} + +// GoVersion returns the go version set on the single module, in module mode, +// or the go.work file in workspace mode. +func (mms *MainModuleSet) GoVersion() string { + if !inWorkspaceMode() { + return modFileGoVersion(mms.ModFile(mms.mustGetSingleMainModule())) + } + v := mms.workFileGoVersion + if v == "" { + // Fall back to 1.18 for go.work files. + v = "1.18" + } + return v +} + +func (mms *MainModuleSet) WorkFileReplaceMap() map[module.Version]module.Version { + return mms.workFileReplaceMap +} + +var MainModules *MainModuleSet type Root int @@ -94,6 +266,7 @@ const ( // in go.mod, edit it before loading. func ModFile() *modfile.File { Init() + modFile := MainModules.ModFile(MainModules.mustGetSingleMainModule()) if modFile == nil { die() } @@ -102,9 +275,38 @@ func ModFile() *modfile.File { func BinDir() string { Init() + if cfg.GOBIN != "" { + return cfg.GOBIN + } + if gopath == "" { + return "" + } return filepath.Join(gopath, "bin") } +// InitWorkfile initializes the workFilePath variable for commands that +// operate in workspace mode. It should not be called by other commands, +// for example 'go mod tidy', that don't operate in workspace mode. +func InitWorkfile() { + switch cfg.WorkFile { + case "off": + workFilePath = "" + case "", "auto": + workFilePath = findWorkspaceFile(base.Cwd()) + default: + if !filepath.IsAbs(cfg.WorkFile) { + base.Fatalf("the path provided to -workfile must be an absolute path") + } + workFilePath = cfg.WorkFile + } +} + +// WorkFilePath returns the path of the go.work file, or "" if not in +// workspace mode. WorkFilePath must be called after InitWorkfile. +func WorkFilePath() string { + return workFilePath +} + // Init determines whether module mode is enabled, locates the root of the // current module (if any), sets environment variables for Git subprocesses, and // configures the cfg, codehost, load, modfetch, and search packages for use @@ -169,18 +371,18 @@ func Init() { if os.Getenv("GCM_INTERACTIVE") == "" { os.Setenv("GCM_INTERACTIVE", "never") } - - if modRoot != "" { + if modRoots != nil { // modRoot set before Init was called ("go mod init" does this). // No need to search for go.mod. } else if RootMode == NoRoot { if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") } - modRoot = "" + modRoots = nil + } else if inWorkspaceMode() { + // We're in workspace mode. } else { - modRoot = findModuleRoot(base.Cwd()) - if modRoot == "" { + if modRoot := findModuleRoot(base.Cwd()); modRoot == "" { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } @@ -198,11 +400,12 @@ func Init() { // will find it and get modules when they're not expecting them. // It's a bit of a peculiar thing to disallow but quite mysterious // when it happens. See golang.org/issue/26708. - modRoot = "" fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) if !mustUseModules { return } + } else { + modRoots = []string{modRoot} } } if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { @@ -213,35 +416,11 @@ func Init() { cfg.ModulesEnabled = true setDefaultBuildMod() list := filepath.SplitList(cfg.BuildContext.GOPATH) - if len(list) == 0 || list[0] == "" { - base.Fatalf("missing $GOPATH") - } - gopath = list[0] - if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { - base.Fatalf("$GOPATH/go.mod exists but should not") - } - - if modRoot == "" { - // We're in module mode, but not inside a module. - // - // Commands like 'go build', 'go run', 'go list' have no go.mod file to - // read or write. They would need to find and download the latest versions - // of a potentially large number of modules with no way to save version - // information. We can succeed slowly (but not reproducibly), but that's - // not usually a good experience. - // - // Instead, we forbid resolving import paths to modules other than std and - // cmd. Users may still build packages specified with .go files on the - // command line, but they'll see an error if those files import anything - // outside std. - // - // This can be overridden by calling AllowMissingModuleImports. - // For example, 'go get' does this, since it is expected to resolve paths. - // - // See golang.org/issue/32027. - } else { - modfetch.GoSumFile = strings.TrimSuffix(ModFilePath(), ".mod") + ".sum" - search.SetModRoot(modRoot) + if len(list) > 0 && list[0] != "" { + gopath = list[0] + if _, err := fsys.Stat(filepath.Join(gopath, "go.mod")); err == nil { + base.Fatalf("$GOPATH/go.mod exists but should not") + } } } @@ -255,7 +434,7 @@ func Init() { // be called until the command is installed and flags are parsed. Instead of // calling Init and Enabled, the main package can call this function. func WillBeEnabled() bool { - if modRoot != "" || cfg.ModulesEnabled { + if modRoots != nil || cfg.ModulesEnabled { // Already enabled. return true } @@ -297,16 +476,18 @@ func WillBeEnabled() bool { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || cfg.ModulesEnabled + return modRoots != nil || cfg.ModulesEnabled } -// ModRoot returns the root of the main module. -// It calls base.Fatalf if there is no main module. -func ModRoot() string { - if !HasModRoot() { - die() +func VendorDir() string { + return filepath.Join(MainModules.ModRoot(MainModules.mustGetSingleMainModule()), "vendor") +} + +func inWorkspaceMode() bool { + if !initialized { + panic("inWorkspaceMode called before modload.Init called") } - return modRoot + return workFilePath != "" } // HasModRoot reports whether a main module is present. @@ -314,17 +495,27 @@ func ModRoot() string { // does not require a main module. func HasModRoot() bool { Init() - return modRoot != "" + return modRoots != nil } -// ModFilePath returns the effective path of the go.mod file. Normally, this -// "go.mod" in the directory returned by ModRoot, but the -modfile flag may -// change its location. ModFilePath calls base.Fatalf if there is no main -// module, even if -modfile is set. -func ModFilePath() string { +// MustHaveModRoot checks that a main module or main modules are present, +// and calls base.Fatalf if there are no main modules. +func MustHaveModRoot() { + Init() if !HasModRoot() { die() } +} + +// ModFilePath returns the path that would be used for the go.mod +// file, if in module mode. ModFilePath calls base.Fatalf if there is no main +// module, even if -modfile is set. +func ModFilePath() string { + MustHaveModRoot() + return modFilePath(findModuleRoot(base.Cwd())) +} + +func modFilePath(modRoot string) string { if cfg.ModFile != "" { return cfg.ModFile } @@ -335,6 +526,9 @@ func die() { if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") } + if inWorkspaceMode() { + base.Fatalf("go: no modules were found in the current workspace; see 'go help work'") + } if dir, name := findAltConfig(base.Cwd()); dir != "" { rel, err := filepath.Rel(base.Cwd(), dir) if err != nil { @@ -365,12 +559,81 @@ func (goModDirtyError) Error() string { var errGoModDirty error = goModDirtyError{} +func loadWorkFile(path string) (goVersion string, modRoots []string, replaces []*modfile.Replace, err error) { + workDir := filepath.Dir(path) + wf, err := ReadWorkFile(path) + if err != nil { + return "", nil, nil, err + } + if wf.Go != nil { + goVersion = wf.Go.Version + } + seen := map[string]bool{} + for _, d := range wf.Use { + modRoot := d.Path + if !filepath.IsAbs(modRoot) { + modRoot = filepath.Join(workDir, modRoot) + } + + if seen[modRoot] { + return "", nil, nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot) + } + seen[modRoot] = true + modRoots = append(modRoots, modRoot) + } + + return goVersion, modRoots, wf.Replace, nil +} + +// ReadWorkFile reads and parses the go.work file at the given path. +func ReadWorkFile(path string) (*modfile.WorkFile, error) { + workData, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + return modfile.ParseWork(path, workData, nil) +} + +// WriteWorkFile cleans and writes out the go.work file to the given path. +func WriteWorkFile(path string, wf *modfile.WorkFile) error { + wf.SortBlocks() + wf.Cleanup() + out := modfile.Format(wf.Syntax) + + return ioutil.WriteFile(path, out, 0666) +} + +// UpdateWorkFile updates comments on directory directives in the go.work +// file to include the associated module path. +func UpdateWorkFile(wf *modfile.WorkFile) { + missingModulePaths := map[string]string{} // module directory listed in file -> abspath modroot + + for _, d := range wf.Use { + modRoot := d.Path + if d.ModulePath == "" { + missingModulePaths[d.Path] = modRoot + } + } + + // Clean up and annotate directories. + // TODO(matloob): update x/mod to actually add module paths. + for moddir, absmodroot := range missingModulePaths { + _, f, err := ReadModFile(filepath.Join(absmodroot, "go.mod"), nil) + if err != nil { + continue // Error will be reported if modules are loaded. + } + wf.AddUse(moddir, f.Module.Mod.Path) + } +} + // LoadModFile sets Target and, if there is a main module, parses the initial // build list from its go.mod file. // // LoadModFile may make changes in memory, like adding a go directive and -// ensuring requirements are consistent, and will write those changes back to -// disk unless DisallowWriteGoMod is in effect. +// ensuring requirements are consistent. The caller is responsible for ensuring +// those changes are written to disk by calling LoadPackages or ListModules +// (unless ExplicitWriteGoMod is set) or by calling WriteGoMod directly. // // As a side-effect, LoadModFile may change cfg.BuildMod to "vendor" if // -mod wasn't set explicitly and automatic vendoring should be enabled. @@ -383,111 +646,142 @@ var errGoModDirty error = goModDirtyError{} // it for global consistency. Most callers outside of the modload package should // use LoadModGraph instead. func LoadModFile(ctx context.Context) *Requirements { - rs, needCommit := loadModFile(ctx) - if needCommit { - commitRequirements(ctx, modFileGoVersion(), rs) - } - return rs -} - -// loadModFile is like LoadModFile, but does not implicitly commit the -// requirements back to disk after fixing inconsistencies. -// -// If needCommit is true, after the caller makes any other needed changes to the -// returned requirements they should invoke commitRequirements to fix any -// inconsistencies that may be present in the on-disk go.mod file. -func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { if requirements != nil { - return requirements, false + return requirements } Init() - if modRoot == "" { - Target = module.Version{Path: "command-line-arguments"} - targetPrefix = "command-line-arguments" - goVersion := LatestGoVersion() - rawGoVersion.Store(Target, goVersion) - requirements = newRequirements(modDepthFromGoVersion(goVersion), nil, nil) - return requirements, false - } - - gomod := ModFilePath() - var data []byte - var err error - if gomodActual, ok := fsys.OverlayPath(gomod); ok { - // Don't lock go.mod if it's part of the overlay. - // On Plan 9, locking requires chmod, and we don't want to modify any file - // in the overlay. See #44700. - data, err = os.ReadFile(gomodActual) + var ( + workFileGoVersion string + workFileReplaces []*modfile.Replace + ) + if inWorkspaceMode() { + var err error + workFileGoVersion, modRoots, workFileReplaces, err = loadWorkFile(workFilePath) + if err != nil { + base.Fatalf("reading go.work: %v", err) + } + for _, modRoot := range modRoots { + sumFile := strings.TrimSuffix(modFilePath(modRoot), ".mod") + ".sum" + modfetch.WorkspaceGoSumFiles = append(modfetch.WorkspaceGoSumFiles, sumFile) + } + modfetch.GoSumFile = workFilePath + ".sum" + } else if modRoots == nil { + // We're in module mode, but not inside a module. + // + // Commands like 'go build', 'go run', 'go list' have no go.mod file to + // read or write. They would need to find and download the latest versions + // of a potentially large number of modules with no way to save version + // information. We can succeed slowly (but not reproducibly), but that's + // not usually a good experience. + // + // Instead, we forbid resolving import paths to modules other than std and + // cmd. Users may still build packages specified with .go files on the + // command line, but they'll see an error if those files import anything + // outside std. + // + // This can be overridden by calling AllowMissingModuleImports. + // For example, 'go get' does this, since it is expected to resolve paths. + // + // See golang.org/issue/32027. } else { - data, err = lockedfile.Read(gomodActual) - } - if err != nil { - base.Fatalf("go: %v", err) + modfetch.GoSumFile = strings.TrimSuffix(modFilePath(modRoots[0]), ".mod") + ".sum" + } + if len(modRoots) == 0 { + // TODO(#49228): Instead of creating a fake module with an empty modroot, + // make MainModules.Len() == 0 mean that we're in module mode but not inside + // any module. + mainModule := module.Version{Path: "command-line-arguments"} + MainModules = makeMainModules([]module.Version{mainModule}, []string{""}, []*modfile.File{nil}, []*modFileIndex{nil}, "", nil) + goVersion := LatestGoVersion() + rawGoVersion.Store(mainModule, goVersion) + pruning := pruningForGoVersion(goVersion) + if inWorkspaceMode() { + pruning = workspace + } + requirements = newRequirements(pruning, nil, nil) + return requirements } - var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) - if err != nil { - // Errors returned by modfile.Parse begin with file:line. - base.Fatalf("go: errors parsing go.mod:\n%s\n", err) - } - if f.Module == nil { - // No module declaration. Must add module path. - base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") - } + var modFiles []*modfile.File + var mainModules []module.Version + var indices []*modFileIndex + for _, modroot := range modRoots { + gomod := modFilePath(modroot) + var fixed bool + data, f, err := ReadModFile(gomod, fixVersion(ctx, &fixed)) + if err != nil { + base.Fatalf("go: %v", err) + } - modFile = f - initTarget(f.Module.Mod) - index = indexModFile(data, f, fixed) + modFiles = append(modFiles, f) + mainModule := f.Module.Mod + mainModules = append(mainModules, mainModule) + indices = append(indices, indexModFile(data, f, mainModule, fixed)) - if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { - if pathErr, ok := err.(*module.InvalidPathError); ok { - pathErr.Kind = "module" + if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { + if pathErr, ok := err.(*module.InvalidPathError); ok { + pathErr.Kind = "module" + } + base.Fatalf("go: %v", err) } - base.Fatalf("go: %v", err) } + MainModules = makeMainModules(mainModules, modRoots, modFiles, indices, workFileGoVersion, workFileReplaces) setDefaultBuildMod() // possibly enable automatic vendoring - rs = requirementsFromModFile() + rs := requirementsFromModFiles(ctx, modFiles) + + if inWorkspaceMode() { + // We don't need to do anything for vendor or update the mod file so + // return early. + requirements = rs + return rs + } + + mainModule := MainModules.mustGetSingleMainModule() + if cfg.BuildMod == "vendor" { - readVendorList() - checkVendorConsistency() + readVendorList(mainModule) + index := MainModules.Index(mainModule) + modFile := MainModules.ModFile(mainModule) + checkVendorConsistency(index, modFile) rs.initVendor(vendorList) } + if rs.hasRedundantRoot() { // If any module path appears more than once in the roots, we know that the // go.mod file needs to be updated even though we have not yet loaded any // transitive dependencies. + var err error rs, err = updateRoots(ctx, rs.direct, rs, nil, nil, false) if err != nil { base.Fatalf("go: %v", err) } } - if index.goVersionV == "" { + if MainModules.Index(mainModule).goVersionV == "" && rs.pruning != workspace { // TODO(#45551): Do something more principled instead of checking // cfg.CmdName directly here. if cfg.BuildMod == "mod" && cfg.CmdName != "mod graph" && cfg.CmdName != "mod why" { - addGoStmt(LatestGoVersion()) - if go117EnableLazyLoading { - // We need to add a 'go' version to the go.mod file, but we must assume - // that its existing contents match something between Go 1.11 and 1.16. - // Go 1.11 through 1.16 have eager requirements, but the latest Go - // version uses lazy requirements instead — so we need to cnvert the - // requirements to be lazy. - rs, err = convertDepth(ctx, rs, lazy) - if err != nil { - base.Fatalf("go: %v", err) - } + addGoStmt(MainModules.ModFile(mainModule), mainModule, LatestGoVersion()) + + // We need to add a 'go' version to the go.mod file, but we must assume + // that its existing contents match something between Go 1.11 and 1.16. + // Go 1.11 through 1.16 do not support graph pruning, but the latest Go + // version uses a pruned module graph — so we need to convert the + // requirements to support pruning. + var err error + rs, err = convertPruning(ctx, rs, pruned) + if err != nil { + base.Fatalf("go: %v", err) } } else { - rawGoVersion.Store(Target, modFileGoVersion()) + rawGoVersion.Store(mainModule, modFileGoVersion(MainModules.ModFile(mainModule))) } } requirements = rs - return requirements, true + return requirements } // CreateModFile initializes a new module by creating a go.mod file. @@ -500,9 +794,10 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { // exactly the same as in the legacy configuration (for example, we can't get // packages at multiple versions from the same module). func CreateModFile(ctx context.Context, modPath string) { - modRoot = base.Cwd() + modRoot := base.Cwd() + modRoots = []string{modRoot} Init() - modFilePath := ModFilePath() + modFilePath := modFilePath(modRoot) if _, err := fsys.Stat(modFilePath); err == nil { base.Fatalf("go: %s already exists", modFilePath) } @@ -523,15 +818,22 @@ func CreateModFile(ctx context.Context, modPath string) { } } base.Fatalf("go: %v", err) + } else if _, _, ok := module.SplitPathVersion(modPath); !ok { + if strings.HasPrefix(modPath, "gopkg.in/") { + invalidMajorVersionMsg := fmt.Errorf("module paths beginning with gopkg.in/ must always have a major version suffix in the form of .vN:\n\tgo mod init %s", suggestGopkgIn(modPath)) + base.Fatalf(`go: invalid module path "%v": %v`, modPath, invalidMajorVersionMsg) + } + invalidMajorVersionMsg := fmt.Errorf("major version suffixes must be in the form of /vN and are only allowed for v2 or later:\n\tgo mod init %s", suggestModulePath(modPath)) + base.Fatalf(`go: invalid module path "%v": %v`, modPath, invalidMajorVersionMsg) } fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) - modFile = new(modfile.File) + modFile := new(modfile.File) modFile.AddModuleStmt(modPath) - initTarget(modFile.Module.Mod) - addGoStmt(LatestGoVersion()) // Add the go directive before converted module requirements. + MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}, "", nil) + addGoStmt(modFile, modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. - convertedFrom, err := convertLegacyConfig(modPath) + convertedFrom, err := convertLegacyConfig(modFile, modRoot) if convertedFrom != "" { fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) } @@ -539,12 +841,15 @@ func CreateModFile(ctx context.Context, modPath string) { base.Fatalf("go: %v", err) } - rs := requirementsFromModFile() + rs := requirementsFromModFiles(ctx, []*modfile.File{modFile}) rs, err = updateRoots(ctx, rs.direct, rs, nil, nil, false) if err != nil { base.Fatalf("go: %v", err) } - commitRequirements(ctx, modFileGoVersion(), rs) + requirements = rs + if err := commitRequirements(ctx); err != nil { + base.Fatalf("go: %v", err) + } // Suggest running 'go mod tidy' unless the project is empty. Even if we // imported all the correct requirements above, we're probably missing @@ -570,6 +875,32 @@ func CreateModFile(ctx context.Context, modPath string) { } } +// CreateWorkFile initializes a new workspace by creating a go.work file. +func CreateWorkFile(ctx context.Context, workFile string, modDirs []string) { + if _, err := fsys.Stat(workFile); err == nil { + base.Fatalf("go: %s already exists", workFile) + } + + goV := LatestGoVersion() // Use current Go version by default + workF := new(modfile.WorkFile) + workF.Syntax = new(modfile.FileSyntax) + workF.AddGoStmt(goV) + + for _, dir := range modDirs { + _, f, err := ReadModFile(filepath.Join(dir, "go.mod"), nil) + if err != nil { + if os.IsNotExist(err) { + base.Fatalf("go: creating workspace file: no go.mod file exists in directory %v", dir) + } + base.Fatalf("go: error parsing go.mod in directory %s: %v", dir, err) + } + workF.AddUse(ToDirectoryPath(dir), f.Module.Mod.Path) + } + + UpdateWorkFile(workF) + WriteWorkFile(workFile, workF) +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, @@ -632,49 +963,125 @@ func AllowMissingModuleImports() { allowMissingModuleImports = true } -// initTarget sets Target and associated variables according to modFile, -func initTarget(m module.Version) { - Target = m - targetPrefix = m.Path - - if rel := search.InDir(base.Cwd(), cfg.GOROOTsrc); rel != "" { - targetInGorootSrc = true - if m.Path == "std" { - // The "std" module in GOROOT/src is the Go standard library. Unlike other - // modules, the packages in the "std" module have no import-path prefix. - // - // Modules named "std" outside of GOROOT/src do not receive this special - // treatment, so it is possible to run 'go test .' in other GOROOTs to - // test individual packages using a combination of the modified package - // and the ordinary standard library. - // (See https://golang.org/issue/30756.) - targetPrefix = "" +// makeMainModules creates a MainModuleSet and associated variables according to +// the given main modules. +func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile.File, indices []*modFileIndex, workFileGoVersion string, workFileReplaces []*modfile.Replace) *MainModuleSet { + for _, m := range ms { + if m.Version != "" { + panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) } } + modRootContainingCWD := findModuleRoot(base.Cwd()) + mainModules := &MainModuleSet{ + versions: ms[:len(ms):len(ms)], + inGorootSrc: map[module.Version]bool{}, + pathPrefix: map[module.Version]string{}, + modRoot: map[module.Version]string{}, + modFiles: map[module.Version]*modfile.File{}, + indices: map[module.Version]*modFileIndex{}, + workFileGoVersion: workFileGoVersion, + workFileReplaceMap: toReplaceMap(workFileReplaces), + highestReplaced: map[string]string{}, + } + mainModulePaths := make(map[string]bool) + for _, m := range ms { + mainModulePaths[m.Path] = true + } + replacedByWorkFile := make(map[string]bool) + replacements := make(map[module.Version]module.Version) + for _, r := range workFileReplaces { + if mainModulePaths[r.Old.Path] && r.Old.Version == "" { + base.Errorf("go: workspace module %v is replaced at all versions in the go.work file. To fix, remove the replacement from the go.work file or specify the version at which to replace the module.", r.Old.Path) + } + replacedByWorkFile[r.Old.Path] = true + v, ok := mainModules.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + mainModules.highestReplaced[r.Old.Path] = r.Old.Version + } + replacements[r.Old] = r.New + } + for i, m := range ms { + mainModules.pathPrefix[m] = m.Path + mainModules.modRoot[m] = rootDirs[i] + mainModules.modFiles[m] = modFiles[i] + mainModules.indices[m] = indices[i] + + if mainModules.modRoot[m] == modRootContainingCWD { + mainModules.modContainingCWD = m + } + + if rel := search.InDir(rootDirs[i], cfg.GOROOTsrc); rel != "" { + mainModules.inGorootSrc[m] = true + if m.Path == "std" { + // The "std" module in GOROOT/src is the Go standard library. Unlike other + // modules, the packages in the "std" module have no import-path prefix. + // + // Modules named "std" outside of GOROOT/src do not receive this special + // treatment, so it is possible to run 'go test .' in other GOROOTs to + // test individual packages using a combination of the modified package + // and the ordinary standard library. + // (See https://golang.org/issue/30756.) + mainModules.pathPrefix[m] = "" + } + } + + if modFiles[i] != nil { + curModuleReplaces := make(map[module.Version]bool) + for _, r := range modFiles[i].Replace { + if replacedByWorkFile[r.Old.Path] { + continue + } else if prev, ok := replacements[r.Old]; ok && !curModuleReplaces[r.Old] && prev != r.New { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v\nuse \"go work edit -replace %v=[override]\" to resolve", r.Old, prev, r.New, r.Old) + } + curModuleReplaces[r.Old] = true + replacements[r.Old] = r.New + + v, ok := mainModules.highestReplaced[r.Old.Path] + if !ok || semver.Compare(r.Old.Version, v) > 0 { + mainModules.highestReplaced[r.Old.Path] = r.Old.Version + } + } + } + } + return mainModules } -// requirementsFromModFile returns the set of non-excluded requirements from +// requirementsFromModFiles returns the set of non-excluded requirements from // the global modFile. -func requirementsFromModFile() *Requirements { - roots := make([]module.Version, 0, len(modFile.Require)) +func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Requirements { + var roots []module.Version direct := map[string]bool{} - for _, r := range modFile.Require { - if index != nil && index.exclude[r.Mod] { - if cfg.BuildMod == "mod" { - fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) - } else { - fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) - } - continue + var pruning modPruning + if inWorkspaceMode() { + pruning = workspace + roots = make([]module.Version, len(MainModules.Versions())) + copy(roots, MainModules.Versions()) + } else { + pruning = pruningForGoVersion(MainModules.GoVersion()) + if len(modFiles) != 1 { + panic(fmt.Errorf("requirementsFromModFiles called with %v modfiles outside workspace mode", len(modFiles))) } + modFile := modFiles[0] + roots = make([]module.Version, 0, len(modFile.Require)) + mm := MainModules.mustGetSingleMainModule() + for _, r := range modFile.Require { + if index := MainModules.Index(mm); index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + continue + } - roots = append(roots, r.Mod) - if !r.Indirect { - direct[r.Mod.Path] = true + roots = append(roots, r.Mod) + if !r.Indirect { + direct[r.Mod.Path] = true + } } } module.Sort(roots) - rs := newRequirements(modDepthFromGoVersion(modFileGoVersion()), roots, direct) + rs := newRequirements(pruning, roots, direct) return rs } @@ -682,18 +1089,35 @@ func requirementsFromModFile() *Requirements { // wasn't provided. setDefaultBuildMod may be called multiple times. func setDefaultBuildMod() { if cfg.BuildModExplicit { + if inWorkspaceMode() && cfg.BuildMod != "readonly" { + base.Fatalf("go: -mod may only be set to readonly when in workspace mode, but it is set to %q"+ + "\n\tRemove the -mod flag to use the default readonly value,"+ + "\n\tor set -workfile=off to disable workspace mode.", cfg.BuildMod) + } // Don't override an explicit '-mod=' argument. return } - if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // 'get' and 'go mod' commands may update go.mod automatically. - // TODO(jayconrod): should this narrower? Should 'go mod download' or - // 'go mod graph' update go.mod by default? + // TODO(#40775): commands should pass in the module mode as an option + // to modload functions instead of relying on an implicit setting + // based on command name. + switch cfg.CmdName { + case "get", "mod download", "mod init", "mod tidy", "work sync": + // These commands are intended to update go.mod and go.sum. + cfg.BuildMod = "mod" + return + case "mod graph", "mod verify", "mod why": + // These commands should not update go.mod or go.sum, but they should be + // able to fetch modules not in go.sum and should not report errors if + // go.mod is inconsistent. They're useful for debugging, and they need + // to work in buggy situations. cfg.BuildMod = "mod" return + case "mod vendor": + cfg.BuildMod = "readonly" + return } - if modRoot == "" { + if modRoots == nil { if allowMissingModuleImports { cfg.BuildMod = "mod" } else { @@ -702,31 +1126,38 @@ func setDefaultBuildMod() { return } - if fi, err := fsys.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { - modGo := "unspecified" - if index != nil && index.goVersionV != "" { - if semver.Compare(index.goVersionV, "v1.14") >= 0 { - // The Go version is at least 1.14, and a vendor directory exists. - // Set -mod=vendor by default. - cfg.BuildMod = "vendor" - cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." - return - } else { - modGo = index.goVersionV[1:] + if len(modRoots) == 1 { + index := MainModules.GetSingleIndexOrNil() + if fi, err := fsys.Stat(filepath.Join(modRoots[0], "vendor")); err == nil && fi.IsDir() { + modGo := "unspecified" + if index != nil && index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + cfg.BuildMod = "vendor" + cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." + return + } else { + modGo = index.goVersionV[1:] + } } - } - // Since a vendor directory exists, we should record why we didn't use it. - // This message won't normally be shown, but it may appear with import errors. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) + } } cfg.BuildMod = "readonly" } +func mustHaveCompleteRequirements() bool { + return cfg.BuildMod != "mod" && !inWorkspaceMode() +} + // convertLegacyConfig imports module requirements from a legacy vendoring // configuration file, if one is present. -func convertLegacyConfig(modPath string) (from string, err error) { +func convertLegacyConfig(modFile *modfile.File, modRoot string) (from string, err error) { noneSelected := func(path string) (version string) { return "none" } queryPackage := func(path, rev string) (module.Version, error) { pkgMods, modOnly, err := QueryPattern(context.Background(), path, rev, noneSelected, nil) @@ -757,14 +1188,14 @@ func convertLegacyConfig(modPath string) (from string, err error) { // addGoStmt adds a go directive to the go.mod file if it does not already // include one. The 'go' version added, if any, is the latest version supported // by this toolchain. -func addGoStmt(v string) { +func addGoStmt(modFile *modfile.File, mod module.Version, v string) { if modFile.Go != nil && modFile.Go.Version != "" { return } if err := modFile.AddGoStmt(v); err != nil { base.Fatalf("go: internal error: %v", err) } - rawGoVersion.Store(Target, v) + rawGoVersion.Store(mod, v) } // LatestGoVersion returns the latest version of the Go language supported by @@ -815,7 +1246,7 @@ var altConfigs = []string{ ".git/config", } -func findModuleRoot(dir string) (root string) { +func findModuleRoot(dir string) (roots string) { if dir == "" { panic("dir not set") } @@ -835,6 +1266,33 @@ func findModuleRoot(dir string) (root string) { return "" } +func findWorkspaceFile(dir string) (root string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + f := filepath.Join(dir, "go.work") + if fi, err := fsys.Stat(f); err == nil && !fi.IsDir() { + return f + } + d := filepath.Dir(dir) + if d == dir { + break + } + if d == cfg.GOROOT { + // As a special case, don't cross GOROOT to find a go.work file. + // The standard library and commands built in go always use the vendored + // dependencies, so avoid using a most likely irrelevant go.work file. + return "" + } + dir = d + } + return "" +} + func findAltConfig(dir string) (root, name string) { if dir == "" { panic("dir not set") @@ -960,66 +1418,62 @@ func findImportComment(file string) string { return path } -var allowWriteGoMod = true - -// DisallowWriteGoMod causes future calls to WriteGoMod to do nothing at all. -func DisallowWriteGoMod() { - allowWriteGoMod = false -} - -// AllowWriteGoMod undoes the effect of DisallowWriteGoMod: -// future calls to WriteGoMod will update go.mod if needed. -// Note that any past calls have been discarded, so typically -// a call to AlowWriteGoMod should be followed by a call to WriteGoMod. -func AllowWriteGoMod() { - allowWriteGoMod = true -} - // WriteGoMod writes the current build list back to go.mod. -func WriteGoMod(ctx context.Context) { - if !allowWriteGoMod { - panic("WriteGoMod called while disallowed") - } - commitRequirements(ctx, modFileGoVersion(), LoadModFile(ctx)) +func WriteGoMod(ctx context.Context) error { + requirements = LoadModFile(ctx) + return commitRequirements(ctx) } -// commitRequirements writes sets the global requirements variable to rs and -// writes its contents back to the go.mod file on disk. -func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) { - requirements = rs - - if !allowWriteGoMod { - // Some package outside of modload promised to update the go.mod file later. - return - } - - if modRoot == "" { +// commitRequirements ensures go.mod and go.sum are up to date with the current +// requirements. +// +// In "mod" mode, commitRequirements writes changes to go.mod and go.sum. +// +// In "readonly" and "vendor" modes, commitRequirements returns an error if +// go.mod or go.sum are out of date in a semantically significant way. +// +// In workspace mode, commitRequirements only writes changes to go.work.sum. +func commitRequirements(ctx context.Context) (err error) { + if inWorkspaceMode() { + // go.mod files aren't updated in workspace mode, but we still want to + // update the go.work.sum file. + return modfetch.WriteGoSum(keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()) + } + if MainModules.Len() != 1 || MainModules.ModRoot(MainModules.Versions()[0]) == "" { // We aren't in a module, so we don't have anywhere to write a go.mod file. - return + return nil + } + mainModule := MainModules.mustGetSingleMainModule() + modFile := MainModules.ModFile(mainModule) + if modFile == nil { + // command-line-arguments has no .mod file to write. + return nil } + modFilePath := modFilePath(MainModules.ModRoot(mainModule)) var list []*modfile.Require - for _, m := range rs.rootModules { + for _, m := range requirements.rootModules { list = append(list, &modfile.Require{ Mod: m, - Indirect: !rs.direct[m.Path], + Indirect: !requirements.direct[m.Path], }) } - if goVersion != "" { - modFile.AddGoStmt(goVersion) + if modFile.Go == nil || modFile.Go.Version == "" { + modFile.AddGoStmt(modFileGoVersion(modFile)) } - if semver.Compare("v"+modFileGoVersion(), separateIndirectVersionV) < 0 { + if semver.Compare("v"+modFileGoVersion(modFile), separateIndirectVersionV) < 0 { modFile.SetRequire(list) } else { modFile.SetRequireSeparateIndirect(list) } modFile.Cleanup() + index := MainModules.GetSingleIndexOrNil() dirty := index.modFileIsDirty(modFile) if dirty && cfg.BuildMod != "mod" { // If we're about to fail due to -mod=readonly, // prefer to report a dirty go.mod over a dirty go.sum - base.Fatalf("go: %v", errGoModDirty) + return errGoModDirty } if !dirty && cfg.CmdName != "mod tidy" { @@ -1028,30 +1482,33 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) // Don't write go.mod, but write go.sum in case we added or trimmed sums. // 'go mod init' shouldn't write go.sum, since it will be incomplete. if cfg.CmdName != "mod init" { - modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums)) + if err := modfetch.WriteGoSum(keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()); err != nil { + return err + } } - return + return nil } - gomod := ModFilePath() - if _, ok := fsys.OverlayPath(gomod); ok { + if _, ok := fsys.OverlayPath(modFilePath); ok { if dirty { - base.Fatalf("go: updates to go.mod needed, but go.mod is part of the overlay specified with -overlay") + return errors.New("updates to go.mod needed, but go.mod is part of the overlay specified with -overlay") } - return + return nil } new, err := modFile.Format() if err != nil { - base.Fatalf("go: %v", err) + return err } defer func() { // At this point we have determined to make the go.mod file on disk equal to new. - index = indexModFile(new, modFile, false) + MainModules.SetIndex(mainModule, indexModFile(new, modFile, mainModule, false)) // Update go.sum after releasing the side lock and refreshing the index. // 'go mod init' shouldn't write go.sum, since it will be incomplete. if cfg.CmdName != "mod init" { - modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums)) + if err == nil { + err = modfetch.WriteGoSum(keepSums(ctx, loaded, requirements, addBuildListZipSums), mustHaveCompleteRequirements()) + } } }() @@ -1063,7 +1520,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) errNoChange := errors.New("no update needed") - err = lockedfile.Transform(ModFilePath(), func(old []byte) ([]byte, error) { + err = lockedfile.Transform(modFilePath, func(old []byte) ([]byte, error) { if bytes.Equal(old, new) { // The go.mod file is already equal to new, possibly as the result of some // other process. @@ -1084,8 +1541,9 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) }) if err != nil && err != errNoChange { - base.Fatalf("go: updating go.mod: %v", err) + return fmt.Errorf("updating go.mod: %w", err) } + return nil } // keepSums returns the set of modules (and go.mod file entries) for which @@ -1113,16 +1571,18 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums continue } - if rs.depth == lazy && pkg.mod.Path != "" { + if rs.pruning == pruned && pkg.mod.Path != "" { if v, ok := rs.rootSelected(pkg.mod.Path); ok && v == pkg.mod.Version { - // pkg was loaded from a root module, and because the main module is - // lazy we do not check non-root modules for conflicts for packages - // that can be found in roots. So we only need the checksums for the - // root modules that may contain pkg, not all possible modules. + // pkg was loaded from a root module, and because the main module has + // a pruned module graph we do not check non-root modules for + // conflicts for packages that can be found in roots. So we only need + // the checksums for the root modules that may contain pkg, not all + // possible modules. for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { if v, ok := rs.rootSelected(prefix); ok && v != "none" { m := module.Version{Path: prefix, Version: v} - keep[resolveReplacement(m)] = true + r := resolveReplacement(m) + keep[r] = true } } continue @@ -1133,15 +1593,15 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { if v := mg.Selected(prefix); v != "none" { m := module.Version{Path: prefix, Version: v} - keep[resolveReplacement(m)] = true + r := resolveReplacement(m) + keep[r] = true } } } } if rs.graph.Load() == nil { - // The module graph was not loaded, possibly because the main module is lazy - // or possibly because we haven't needed to load the graph yet. + // We haven't needed to load the module graph so far. // Save sums for the root modules (or their replacements), but don't // incur the cost of loading the graph just to find and retain the sums. for _, m := range rs.rootModules { @@ -1158,13 +1618,15 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums // The requirements from m's go.mod file are present in the module graph, // so they are relevant to the MVS result regardless of whether m was // actually selected. - keep[modkey(resolveReplacement(m))] = true + r := resolveReplacement(m) + keep[modkey(r)] = true } }) if which == addBuildListZipSums { for _, m := range mg.BuildList() { - keep[resolveReplacement(m)] = true + r := resolveReplacement(m) + keep[r] = true } } } @@ -1184,3 +1646,56 @@ const ( func modkey(m module.Version) module.Version { return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} } + +func suggestModulePath(path string) string { + var m string + + i := len(path) + for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9' || path[i-1] == '.') { + i-- + } + url := path[:i] + url = strings.TrimSuffix(url, "/v") + url = strings.TrimSuffix(url, "/") + + f := func(c rune) bool { + return c > '9' || c < '0' + } + s := strings.FieldsFunc(path[i:], f) + if len(s) > 0 { + m = s[0] + } + m = strings.TrimLeft(m, "0") + if m == "" || m == "1" { + return url + "/v2" + } + + return url + "/v" + m +} + +func suggestGopkgIn(path string) string { + var m string + i := len(path) + for i > 0 && (('0' <= path[i-1] && path[i-1] <= '9') || (path[i-1] == '.')) { + i-- + } + url := path[:i] + url = strings.TrimSuffix(url, ".v") + url = strings.TrimSuffix(url, "/v") + url = strings.TrimSuffix(url, "/") + + f := func(c rune) bool { + return c > '9' || c < '0' + } + s := strings.FieldsFunc(path, f) + if len(s) > 0 { + m = s[0] + } + + m = strings.TrimLeft(m, "0") + + if m == "" { + return url + ".v1" + } + return url + ".v" + m +} diff --git a/libgo/go/cmd/go/internal/modload/list.go b/libgo/go/cmd/go/internal/modload/list.go index ccdeb9b1d11..f782cd93db3 100644 --- a/libgo/go/cmd/go/internal/modload/list.go +++ b/libgo/go/cmd/go/internal/modload/list.go @@ -72,14 +72,21 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo. } if err == nil { - commitRequirements(ctx, modFileGoVersion(), rs) + requirements = rs + if !ExplicitWriteGoMod { + err = commitRequirements(ctx) + } } return mods, err } func listModules(ctx context.Context, rs *Requirements, args []string, mode ListMode) (_ *Requirements, mods []*modinfo.ModulePublic, mgErr error) { if len(args) == 0 { - return rs, []*modinfo.ModulePublic{moduleInfo(ctx, rs, Target, mode)}, nil + var ms []*modinfo.ModulePublic + for _, m := range MainModules.Versions() { + ms = append(ms, moduleInfo(ctx, rs, m, mode)) + } + return rs, ms, nil } needFullGraph := false @@ -101,7 +108,7 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List path := arg[:i] vers := arg[i+1:] if vers == "upgrade" || vers == "patch" { - if _, ok := rs.rootSelected(path); !ok || rs.depth == eager { + if _, ok := rs.rootSelected(path); !ok || rs.pruning == unpruned { needFullGraph = true if !HasModRoot() { base.Fatalf("go: cannot match %q: %v", arg, ErrNoModRoot) @@ -110,7 +117,7 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List } continue } - if _, ok := rs.rootSelected(arg); !ok || rs.depth == eager { + if _, ok := rs.rootSelected(arg); !ok || rs.pruning == unpruned { needFullGraph = true if mode&ListVersions == 0 && !HasModRoot() { base.Fatalf("go: cannot match %q without -versions or an explicit version: %v", arg, ErrNoModRoot) diff --git a/libgo/go/cmd/go/internal/modload/load.go b/libgo/go/cmd/go/internal/modload/load.go index bce9ad85f42..617b634d263 100644 --- a/libgo/go/cmd/go/internal/modload/load.go +++ b/libgo/go/cmd/go/internal/modload/load.go @@ -40,9 +40,10 @@ package modload // - the main module specifies a go version ≤ 1.15, and the package is imported // by a *test of* another package in "all". // -// When we implement lazy loading, we will record the modules providing packages -// in "all" even when we are only loading individual packages, so we set the -// pkgInAll flag regardless of the whether the "all" pattern is a root. +// When graph pruning is in effect, we want to spot-check the graph-pruning +// invariants — which depend on which packages are known to be in "all" — even +// when we are only loading individual packages, so we set the pkgInAll flag +// regardless of the whether the "all" pattern is a root. // (This is necessary to maintain the “import invariant” described in // https://golang.org/design/36460-lazy-module-loading.) // @@ -230,6 +231,9 @@ type PackageOpts struct { // SilenceUnmatchedWarnings suppresses the warnings normally emitted for // patterns that did not match any packages. SilenceUnmatchedWarnings bool + + // Resolve the query against this module. + MainModule module.Version } // LoadPackages identifies the set of packages matching the given patterns and @@ -255,7 +259,11 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma case m.IsLocal(): // Evaluate list of file system directories on first iteration. if m.Dirs == nil { - matchLocalDirs(ctx, m, rs) + matchModRoots := modRoots + if opts.MainModule != (module.Version{}) { + matchModRoots = []string{MainModules.ModRoot(opts.MainModule)} + } + matchLocalDirs(ctx, matchModRoots, m, rs) } // Make a copy of the directory list and translate to import paths. @@ -274,7 +282,9 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // If we're outside of a module, ensure that the failure mode // indicates that. - ModRoot() + if !HasModRoot() { + die() + } if ld != nil { m.AddError(err) @@ -306,7 +316,11 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // The initial roots are the packages in the main module. // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) + matchModules := MainModules.Versions() + if opts.MainModule != (module.Version{}) { + matchModules = []module.Version{opts.MainModule} + } + matchPackages(ctx, m, opts.Tags, omitStd, matchModules) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -324,7 +338,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } } - initialRS, _ := loadModFile(ctx) // Ignore needCommit — we're going to commit at the end regardless. + initialRS := LoadModFile(ctx) ld := loadFromRoots(ctx, loaderParams{ PackageOpts: opts, @@ -365,7 +379,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma for _, m := range initialRS.rootModules { var unused bool - if ld.requirements.depth == eager { + if ld.requirements.pruning == unpruned { // m is unused if it was dropped from the module graph entirely. If it // was only demoted from direct to indirect, it may still be in use via // a transitive import. @@ -384,7 +398,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } keep := keepSums(ctx, ld, ld.requirements, loadedZipSumsOnly) - if compatDepth := modDepthFromGoVersion(ld.TidyCompatibleVersion); compatDepth != ld.requirements.depth { + if compatDepth := pruningForGoVersion(ld.TidyCompatibleVersion); compatDepth != ld.requirements.pruning { compatRS := newRequirements(compatDepth, ld.requirements.rootModules, ld.requirements.direct) ld.checkTidyCompatibility(ctx, compatRS) @@ -393,7 +407,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } } - if allowWriteGoMod { + if !ExplicitWriteGoMod { modfetch.TrimGoSum(keep) // commitRequirements below will also call WriteGoSum, but the "keep" map @@ -401,13 +415,24 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // loaded.requirements, but here we may have also loaded (and want to // preserve checksums for) additional entities from compatRS, which are // only needed for compatibility with ld.TidyCompatibleVersion. - modfetch.WriteGoSum(keep) + if err := modfetch.WriteGoSum(keep, mustHaveCompleteRequirements()); err != nil { + base.Fatalf("go: %v", err) + } + } + + // Update the go.mod file's Go version if necessary. + modFile := MainModules.ModFile(MainModules.mustGetSingleMainModule()) + if ld.GoVersion != "" { + modFile.AddGoStmt(ld.GoVersion) } } // Success! Update go.mod and go.sum (if needed) and return the results. + // We'll skip updating if ExplicitWriteGoMod is true (the caller has opted + // to call WriteGoMod itself) or if ResolveMissingImports is false (the + // command wants to examine the package graph as-is). loaded = ld - commitRequirements(ctx, loaded.GoVersion, loaded.requirements) + requirements = loaded.requirements for _, pkg := range ld.pkgs { if !pkg.isTest() { @@ -415,12 +440,19 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma } } sort.Strings(loadedPackages) + + if !ExplicitWriteGoMod && opts.ResolveMissingImports { + if err := commitRequirements(ctx); err != nil { + base.Fatalf("go: %v", err) + } + } + return matches, loadedPackages } // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories // outside of the standard library and active modules. -func matchLocalDirs(ctx context.Context, m *search.Match, rs *Requirements) { +func matchLocalDirs(ctx context.Context, modRoots []string, m *search.Match, rs *Requirements) { if !m.IsLocal() { panic(fmt.Sprintf("internal error: resolveLocalDirs on non-local pattern %s", m.Pattern())) } @@ -436,14 +468,23 @@ func matchLocalDirs(ctx context.Context, m *search.Match, rs *Requirements) { if !filepath.IsAbs(dir) { absDir = filepath.Join(base.Cwd(), dir) } - if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(ctx, absDir, rs) == "" { + + modRoot := findModuleRoot(absDir) + found := false + for _, mainModuleRoot := range modRoots { + if mainModuleRoot == modRoot { + found = true + break + } + } + if !found && search.InDir(absDir, cfg.GOROOTsrc) == "" && pathInModuleCache(ctx, absDir, rs) == "" { m.Dirs = []string{} m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) return } } - m.MatchDirs() + m.MatchDirs(modRoots) } // resolveLocalPackage resolves a filesystem path to a package path. @@ -485,49 +526,69 @@ func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (str } } - if modRoot != "" && absDir == modRoot { - if absDir == cfg.GOROOTsrc { - return "", errPkgIsGorootSrc + for _, mod := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mod) + if modRoot != "" && absDir == modRoot { + if absDir == cfg.GOROOTsrc { + return "", errPkgIsGorootSrc + } + return MainModules.PathPrefix(mod), nil } - return targetPrefix, nil } // Note: The checks for @ here are just to avoid misinterpreting // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). // It's not strictly necessary but helpful to keep the checks. - if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { - suffix := filepath.ToSlash(absDir[len(modRoot):]) - if strings.HasPrefix(suffix, "/vendor/") { - if cfg.BuildMod != "vendor" { - return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + var pkgNotFoundErr error + pkgNotFoundLongestPrefix := "" + for _, mainModule := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mainModule) + if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { + suffix := filepath.ToSlash(absDir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + if cfg.BuildMod != "vendor" { + return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + } + + readVendorList(mainModule) + pkg := strings.TrimPrefix(suffix, "/vendor/") + if _, ok := vendorPkgModule[pkg]; !ok { + return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + } + return pkg, nil } - readVendorList() - pkg := strings.TrimPrefix(suffix, "/vendor/") - if _, ok := vendorPkgModule[pkg]; !ok { - return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + mainModulePrefix := MainModules.PathPrefix(mainModule) + if mainModulePrefix == "" { + pkg := strings.TrimPrefix(suffix, "/") + if pkg == "builtin" { + // "builtin" is a pseudo-package with a real source file. + // It's not included in "std", so it shouldn't resolve from "." + // within module "std" either. + return "", errPkgIsBuiltin + } + return pkg, nil } - return pkg, nil - } - if targetPrefix == "" { - pkg := strings.TrimPrefix(suffix, "/") - if pkg == "builtin" { - // "builtin" is a pseudo-package with a real source file. - // It's not included in "std", so it shouldn't resolve from "." - // within module "std" either. - return "", errPkgIsBuiltin + pkg := mainModulePrefix + suffix + if _, ok, err := dirInModule(pkg, mainModulePrefix, modRoot, true); err != nil { + return "", err + } else if !ok { + // This main module could contain the directory but doesn't. Other main + // modules might contain the directory, so wait till we finish the loop + // to see if another main module contains directory. But if not, + // return an error. + if len(mainModulePrefix) > len(pkgNotFoundLongestPrefix) { + pkgNotFoundLongestPrefix = mainModulePrefix + pkgNotFoundErr = &PackageNotInModuleError{MainModules: []module.Version{mainModule}, Pattern: pkg} + } + continue } return pkg, nil } - - pkg := targetPrefix + suffix - if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil { - return "", err - } else if !ok { - return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg} - } - return pkg, nil + } + if pkgNotFoundErr != nil { + return "", pkgNotFoundErr } if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { @@ -560,7 +621,7 @@ func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string if repl := Replacement(m); repl.Path != "" && repl.Version == "" { root = repl.Path if !filepath.IsAbs(root) { - root = filepath.Join(ModRoot(), root) + root = filepath.Join(replaceRelativeTo(), root) } } else if repl.Path != "" { root, err = modfetch.DownloadDir(repl) @@ -583,7 +644,7 @@ func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string return path.Join(m.Path, filepath.ToSlash(sub)), true } - if rs.depth == lazy { + if rs.pruning == pruned { for _, m := range rs.rootModules { if v, _ := rs.rootSelected(m.Path); v != m.Version { continue // m is a root, but we have a higher root for the same path. @@ -596,9 +657,9 @@ func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string } } - // None of the roots contained dir, or we're in eager mode and want to load - // the full module graph more aggressively. Either way, check the full graph - // to see if the directory is a non-root dependency. + // None of the roots contained dir, or the graph is unpruned (so we don't want + // to distinguish between roots and transitive dependencies). Either way, + // check the full graph to see if the directory is a non-root dependency. // // If the roots are not consistent with the full module graph, the selected // versions of root modules may differ from what we already checked above. @@ -645,14 +706,14 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { return roots }, }) - commitRequirements(ctx, loaded.GoVersion, loaded.requirements) + requirements = loaded.requirements } // DirImportPath returns the effective import path for dir, -// provided it is within the main module, or else returns ".". -func DirImportPath(ctx context.Context, dir string) string { +// provided it is within a main module, or else returns ".". +func (mms *MainModuleSet) DirImportPath(ctx context.Context, dir string) (path string, m module.Version) { if !HasModRoot() { - return "." + return ".", module.Version{} } LoadModFile(ctx) // Sets targetPrefix. @@ -662,17 +723,32 @@ func DirImportPath(ctx context.Context, dir string) string { dir = filepath.Clean(dir) } - if dir == modRoot { - return targetPrefix - } - if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { - suffix := filepath.ToSlash(dir[len(modRoot):]) - if strings.HasPrefix(suffix, "/vendor/") { - return strings.TrimPrefix(suffix, "/vendor/") + var longestPrefix string + var longestPrefixPath string + var longestPrefixVersion module.Version + for _, v := range mms.Versions() { + modRoot := mms.ModRoot(v) + if dir == modRoot { + return mms.PathPrefix(v), v + } + if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { + pathPrefix := MainModules.PathPrefix(v) + if pathPrefix > longestPrefix { + longestPrefix = pathPrefix + longestPrefixVersion = v + suffix := filepath.ToSlash(dir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + longestPrefixPath = strings.TrimPrefix(suffix, "/vendor/") + } + longestPrefixPath = mms.PathPrefix(v) + suffix + } } - return targetPrefix + suffix } - return "." + if len(longestPrefix) > 0 { + return longestPrefixPath, longestPrefixVersion + } + + return ".", module.Version{} } // ImportMap returns the actual package import path @@ -783,7 +859,7 @@ func (ld *loader) reset() { // errorf reports an error via either os.Stderr or base.Errorf, // according to whether ld.AllowErrors is set. -func (ld *loader) errorf(format string, args ...interface{}) { +func (ld *loader) errorf(format string, args ...any) { if ld.AllowErrors { fmt.Fprintf(os.Stderr, format, args...) } else { @@ -807,6 +883,7 @@ type loadPkg struct { imports []*loadPkg // packages imported by this one testImports []string // test-only imports, saved for use by pkg.test. inStd bool + altMods []module.Version // modules that could have contained the package but did not // Populated by (*loader).pkgTest: testOnce sync.Once @@ -894,10 +971,7 @@ func (pkg *loadPkg) fromExternalModule() bool { if pkg.mod.Path == "" { return false // loaded from the standard library, not a module } - if pkg.mod.Path == Target.Path { - return false // loaded from the main module. - } - return true + return !MainModules.Contains(pkg.mod.Path) } var errMissing = errors.New("cannot find package") @@ -915,10 +989,10 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { } if ld.GoVersion == "" { - ld.GoVersion = modFileGoVersion() + ld.GoVersion = MainModules.GoVersion() if ld.Tidy && semver.Compare("v"+ld.GoVersion, "v"+LatestGoVersion()) > 0 { - ld.errorf("go mod tidy: go.mod file indicates go %s, but maximum supported version is %s\n", ld.GoVersion, LatestGoVersion()) + ld.errorf("go: go.mod file indicates go %s, but maximum version supported by tidy is %s\n", ld.GoVersion, LatestGoVersion()) base.ExitIfErrors() } } @@ -935,18 +1009,30 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { } if semver.Compare("v"+ld.GoVersion, narrowAllVersionV) < 0 && !ld.UseVendorAll { - // The module's go version explicitly predates the change in "all" for lazy - // loading, so continue to use the older interpretation. + // The module's go version explicitly predates the change in "all" for graph + // pruning, so continue to use the older interpretation. ld.allClosesOverTests = true } var err error - ld.requirements, err = convertDepth(ctx, ld.requirements, modDepthFromGoVersion(ld.GoVersion)) + desiredPruning := pruningForGoVersion(ld.GoVersion) + if ld.requirements.pruning == workspace { + desiredPruning = workspace + } + ld.requirements, err = convertPruning(ctx, ld.requirements, desiredPruning) if err != nil { ld.errorf("go: %v\n", err) } - if ld.requirements.depth == eager { + if ld.requirements.pruning == unpruned { + // If the module graph does not support pruning, we assume that we will need + // the full module graph in order to load package dependencies. + // + // This might not be strictly necessary, but it matches the historical + // behavior of the 'go' command and keeps the go.mod file more consistent in + // case of erroneous hand-edits — which are less likely to be detected by + // spot-checks in modules that do not maintain the expanded go.mod + // requirements needed for graph pruning. var err error ld.requirements, _, err = expandGraph(ctx, ld.requirements) if err != nil { @@ -963,7 +1049,7 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { // build list we're using. rootPkgs := ld.listRoots(ld.requirements) - if ld.requirements.depth == lazy && cfg.BuildMod == "mod" { + if ld.requirements.pruning == pruned && cfg.BuildMod == "mod" { // Before we start loading transitive imports of packages, locate all of // the root packages and promote their containing modules to root modules // dependencies. If their go.mod files are tidy (the common case) and the @@ -1005,7 +1091,7 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { break } if changed { - // Don't resolve missing imports until the module graph have stabilized. + // Don't resolve missing imports until the module graph has stabilized. // If the roots are still changing, they may turn out to specify a // requirement on the missing package(s), and we would rather use a // version specified by a new root than add a new dependency on an @@ -1074,15 +1160,15 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { ld.errorf("go: %v\n", err) } - if ld.requirements.depth == lazy { + if ld.requirements.pruning == pruned { // We continuously add tidy roots to ld.requirements during loading, so at // this point the tidy roots should be a subset of the roots of // ld.requirements, ensuring that no new dependencies are brought inside - // the lazy-loading horizon. + // the graph-pruning horizon. // If that is not the case, there is a bug in the loading loop above. for _, m := range rs.rootModules { if v, ok := ld.requirements.rootSelected(m.Path); !ok || v != m.Version { - ld.errorf("go mod tidy: internal error: a requirement on %v is needed but was not added during package loading\n", m) + ld.errorf("go: internal error: a requirement on %v is needed but was not added during package loading\n", m) base.ExitIfErrors() } } @@ -1124,8 +1210,7 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { } // updateRequirements ensures that ld.requirements is consistent with the -// information gained from ld.pkgs and includes the modules in add as roots at -// at least the given versions. +// information gained from ld.pkgs. // // In particular: // @@ -1168,7 +1253,7 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err } for _, pkg := range ld.pkgs { - if pkg.mod != Target { + if pkg.mod.Version != "" || !MainModules.Contains(pkg.mod.Path) { continue } for _, dep := range pkg.imports { @@ -1176,6 +1261,24 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err continue } + if inWorkspaceMode() { + // In workspace mode / workspace pruning mode, the roots are the main modules + // rather than the main module's direct dependencies. The check below on the selected + // roots does not apply. + if mg, err := rs.Graph(ctx); err != nil { + return false, err + } else if _, ok := mg.RequiredBy(dep.mod); !ok { + // dep.mod is not an explicit dependency, but needs to be. + // See comment on error returned below. + pkg.err = &DirectImportFromImplicitDependencyError{ + ImporterPath: pkg.path, + ImportedPath: dep.path, + Module: dep.mod, + } + } + continue + } + if pkg.err == nil && cfg.BuildMod != "mod" { if v, ok := rs.rootSelected(dep.mod.Path); !ok || v != dep.mod.Version { // dep.mod is not an explicit dependency, but needs to be. @@ -1206,14 +1309,14 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err var addRoots []module.Version if ld.Tidy { - // When we are tidying a lazy module, we may need to add roots to preserve - // the versions of indirect, test-only dependencies that are upgraded - // above or otherwise missing from the go.mod files of direct - // dependencies. (For example, the direct dependency might be a very + // When we are tidying a module with a pruned dependency graph, we may need + // to add roots to preserve the versions of indirect, test-only dependencies + // that are upgraded above or otherwise missing from the go.mod files of + // direct dependencies. (For example, the direct dependency might be a very // stable codebase that predates modules and thus lacks a go.mod file, or - // the author of the direct dependency may have forgotten to commit a - // change to the go.mod file, or may have made an erroneous hand-edit that - // causes it to be untidy.) + // the author of the direct dependency may have forgotten to commit a change + // to the go.mod file, or may have made an erroneous hand-edit that causes + // it to be untidy.) // // Promoting an indirect dependency to a root adds the next layer of its // dependencies to the module graph, which may increase the selected @@ -1283,7 +1386,7 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err // // In some sense, we can think of this as ‘upgraded the module providing // pkg.path from "none" to a version higher than "none"’. - if _, _, err = importFromModules(ctx, pkg.path, rs, nil); err == nil { + if _, _, _, err = importFromModules(ctx, pkg.path, rs, nil); err == nil { changed = true break } @@ -1327,6 +1430,15 @@ func (ld *loader) resolveMissingImports(ctx context.Context) (modAddedBy map[mod var err error mod, err = queryImport(ctx, pkg.path, ld.requirements) if err != nil { + var ime *ImportMissingError + if errors.As(err, &ime) { + for curstack := pkg.stack; curstack != nil; curstack = curstack.stack { + if MainModules.Contains(curstack.mod.Path) { + ime.ImportingMainModule = curstack.mod + break + } + } + } // pkg.err was already non-nil, so we can reasonably attribute the error // for pkg to either the original error or the one returned by // queryImport. The existing error indicates only that we couldn't find @@ -1380,7 +1492,7 @@ func (ld *loader) pkg(ctx context.Context, path string, flags loadPkgFlags) *loa panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") } - pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := ld.pkgCache.Do(path, func() any { pkg := &loadPkg{ path: path, } @@ -1425,7 +1537,7 @@ func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkg // so it's ok if we call it more than is strictly necessary. wantTest := false switch { - case ld.allPatternIsRoot && pkg.mod == Target: + case ld.allPatternIsRoot && MainModules.Contains(pkg.mod.Path): // We are loading the "all" pattern, which includes packages imported by // tests in the main module. This package is in the main module, so we // need to identify the imports of its test even if LoadTests is not set. @@ -1446,7 +1558,7 @@ func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkg if wantTest { var testFlags loadPkgFlags - if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + if MainModules.Contains(pkg.mod.Path) || (ld.allClosesOverTests && new.has(pkgInAll)) { // Tests of packages in the main module are in "all", in the sense that // they cause the packages they import to also be in "all". So are tests // of packages in "all" if "all" closes over test dependencies. @@ -1485,7 +1597,7 @@ func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (ch // If the main module is tidy and the package is in "all" — or if we're // lucky — we can identify all of its imports without actually loading the // full module graph. - m, _, err := importFromModules(ctx, path, ld.requirements, nil) + m, _, _, err := importFromModules(ctx, path, ld.requirements, nil) if err != nil { var missing *ImportMissingError if errors.As(err, &missing) && ld.ResolveMissingImports { @@ -1511,7 +1623,8 @@ func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (ch // module to a root to ensure that any other packages this package // imports are resolved from correct dependency versions. // - // (This is the “argument invariant” from the lazy loading design.) + // (This is the “argument invariant” from + // https://golang.org/design/36460-lazy-module-loading.) need := <-needc need[m] = true needc <- need @@ -1573,7 +1686,7 @@ func (ld *loader) load(ctx context.Context, pkg *loadPkg) { } var mg *ModuleGraph - if ld.requirements.depth == eager { + if ld.requirements.pruning == unpruned { var err error mg, err = ld.requirements.Graph(ctx) if err != nil { @@ -1589,11 +1702,11 @@ func (ld *loader) load(ctx context.Context, pkg *loadPkg) { } } - pkg.mod, pkg.dir, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg) + pkg.mod, pkg.dir, pkg.altMods, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg) if pkg.dir == "" { return } - if pkg.mod == Target { + if MainModules.Contains(pkg.mod.Path) { // Go ahead and mark pkg as in "all". This provides the invariant that a // package that is *only* imported by other packages in "all" is always // marked as such before loading its imports. @@ -1698,13 +1811,14 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if !ld.VendorModulesInGOROOTSrc || Target.Path != "cmd" { + if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("cmd") { vendorPath := pathpkg.Join("cmd", "vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if !ld.VendorModulesInGOROOTSrc || Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + } else if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("std") || str.HasPathPrefix(parentPath, "vendor") { // If we are outside of the 'std' module, resolve imports from within 'std' // to the vendor directory. // @@ -1781,7 +1895,7 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) fmt.Fprintln(os.Stderr) goFlag := "" - if ld.GoVersion != modFileGoVersion() { + if ld.GoVersion != MainModules.GoVersion() { goFlag = " -go=" + ld.GoVersion } @@ -1813,7 +1927,7 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) mg, err := rs.Graph(ctx) if err != nil { - ld.errorf("go mod tidy: error loading go %s module graph: %v\n", ld.TidyCompatibleVersion, err) + ld.errorf("go: error loading go %s module graph: %v\n", ld.TidyCompatibleVersion, err) suggestFixes() return } @@ -1847,7 +1961,7 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) pkg := pkg ld.work.Add(func() { - mod, _, err := importFromModules(ctx, pkg.path, rs, mg) + mod, _, _, err := importFromModules(ctx, pkg.path, rs, mg) if mod != pkg.mod { mismatches := <-mismatchMu mismatches[pkg] = mismatch{mod: mod, err: err} @@ -1900,9 +2014,10 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) case mismatch.err != nil: // pkg resolved successfully, but errors out using the requirements in rs. // - // This could occur because the import is provided by a single lazy root - // (and is thus unambiguous in lazy mode) and also one or more - // transitive dependencies (and is ambiguous in eager mode). + // This could occur because the import is provided by a single root (and + // is thus unambiguous in a main module with a pruned module graph) and + // also one or more transitive dependencies (and is ambiguous with an + // unpruned graph). // // It could also occur because some transitive dependency upgrades the // module that previously provided the package to a version that no @@ -1940,18 +2055,18 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) } case pkg.err != nil: - // pkg had an error in lazy mode (presumably suppressed with the -e flag), - // but not in eager mode. + // pkg had an error in with a pruned module graph (presumably suppressed + // with the -e flag), but the error went away using an unpruned graph. // - // This is possible, if, say, the import is unresolved in lazy mode + // This is possible, if, say, the import is unresolved in the pruned graph // (because the "latest" version of each candidate module either is - // unavailable or does not contain the package), but is resolved in - // eager mode due to a newer-than-latest dependency that is normally - // runed out of the module graph. + // unavailable or does not contain the package), but is resolved in the + // unpruned graph due to a newer-than-latest dependency that is normally + // pruned out. // // This could also occur if the source code for the module providing the - // package in lazy mode has a checksum error, but eager mode upgrades - // that module to a version with a correct checksum. + // package in the pruned graph has a checksum error, but the unpruned + // graph upgrades that module to a version with a correct checksum. // // pkg.err should have already been logged elsewhere — along with a // stack trace — so log only the import path and non-error info here. diff --git a/libgo/go/cmd/go/internal/modload/modfile.go b/libgo/go/cmd/go/internal/modload/modfile.go index 03e02e73b63..627cf1dbc06 100644 --- a/libgo/go/cmd/go/internal/modload/modfile.go +++ b/libgo/go/cmd/go/internal/modload/modfile.go @@ -33,10 +33,13 @@ const ( // tests outside of the main module. narrowAllVersionV = "v1.16" - // lazyLoadingVersionV is the Go version (plus leading "v") at which a + // ExplicitIndirectVersionV is the Go version (plus leading "v") at which a // module's go.mod file is expected to list explicit requirements on every // module that provides any package transitively imported by that module. - lazyLoadingVersionV = "v1.17" + // + // Other indirect dependencies of such a module can be safely pruned out of + // the module graph; see https://golang.org/ref/mod#graph-pruning. + ExplicitIndirectVersionV = "v1.17" // separateIndirectVersionV is the Go version (plus leading "v") at which // "// indirect" dependencies are added in a block separate from the direct @@ -44,23 +47,37 @@ const ( separateIndirectVersionV = "v1.17" ) -const ( - // go117EnableLazyLoading toggles whether lazy-loading code paths should be - // active. It will be removed once the lazy loading implementation is stable - // and well-tested. - go117EnableLazyLoading = true - - // go1117LazyTODO is a constant that exists only until lazy loading is - // implemented. Its use indicates a condition that will need to change if the - // main module is lazy. - go117LazyTODO = false -) +// ReadModFile reads and parses the mod file at gomod. ReadModFile properly applies the +// overlay, locks the file while reading, and applies fix, if applicable. +func ReadModFile(gomod string, fix modfile.VersionFixer) (data []byte, f *modfile.File, err error) { + if gomodActual, ok := fsys.OverlayPath(gomod); ok { + // Don't lock go.mod if it's part of the overlay. + // On Plan 9, locking requires chmod, and we don't want to modify any file + // in the overlay. See #44700. + data, err = os.ReadFile(gomodActual) + } else { + data, err = lockedfile.Read(gomodActual) + } + if err != nil { + return nil, nil, err + } -var modFile *modfile.File + f, err = modfile.Parse(gomod, data, fix) + if err != nil { + // Errors returned by modfile.Parse begin with file:line. + return nil, nil, fmt.Errorf("errors parsing go.mod:\n%s\n", err) + } + if f.Module == nil { + // No module declaration. Must add module path. + return nil, nil, errors.New("no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") + } + + return data, f, err +} // modFileGoVersion returns the (non-empty) Go version at which the requirements -// in modFile are intepreted, or the latest Go version if modFile is nil. -func modFileGoVersion() string { +// in modFile are interpreted, or the latest Go version if modFile is nil. +func modFileGoVersion(modFile *modfile.File) string { if modFile == nil { return LatestGoVersion() } @@ -71,9 +88,9 @@ func modFileGoVersion() string { // has been erroneously hand-edited. // // The semantics of the go.mod file are more-or-less the same from Go 1.11 - // through Go 1.16, changing at 1.17 for lazy loading. So even though a - // go.mod file without a 'go' directive is theoretically a Go 1.11 file, - // scripts may assume that it ends up as a Go 1.16 module. + // through Go 1.16, changing at 1.17 to support module graph pruning. + // So even though a go.mod file without a 'go' directive is theoretically a + // Go 1.11 file, scripts may assume that it ends up as a Go 1.16 module. return "1.16" } return modFile.Go.Version @@ -82,39 +99,37 @@ func modFileGoVersion() string { // A modFileIndex is an index of data corresponding to a modFile // at a specific point in time. type modFileIndex struct { - data []byte - dataNeedsFix bool // true if fixVersion applied a change while parsing data - module module.Version - goVersionV string // GoVersion with "v" prefix - require map[module.Version]requireMeta - replace map[module.Version]module.Version - highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements - exclude map[module.Version]bool + data []byte + dataNeedsFix bool // true if fixVersion applied a change while parsing data + module module.Version + goVersionV string // GoVersion with "v" prefix + require map[module.Version]requireMeta + replace map[module.Version]module.Version + exclude map[module.Version]bool } -// index is the index of the go.mod file as of when it was last read or written. -var index *modFileIndex - type requireMeta struct { indirect bool } -// A modDepth indicates which dependencies should be loaded for a go.mod file. -type modDepth uint8 +// A modPruning indicates whether transitive dependencies of Go 1.17 dependencies +// are pruned out of the module subgraph rooted at a given module. +// (See https://golang.org/ref/mod#graph-pruning.) +type modPruning uint8 const ( - lazy modDepth = iota // load dependencies only as needed - eager // load all transitive dependencies eagerly + pruned modPruning = iota // transitive dependencies of modules at go 1.17 and higher are pruned out + unpruned // no transitive dependencies are pruned out + workspace // pruned to the union of modules in the workspace ) -func modDepthFromGoVersion(goVersion string) modDepth { - if !go117EnableLazyLoading { - return eager - } - if semver.Compare("v"+goVersion, lazyLoadingVersionV) < 0 { - return eager +func pruningForGoVersion(goVersion string) modPruning { + if semver.Compare("v"+goVersion, ExplicitIndirectVersionV) < 0 { + // The go.mod file does not duplicate relevant information about transitive + // dependencies, so they cannot be pruned out. + return unpruned } - return lazy + return pruned } // CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by @@ -137,8 +152,10 @@ var ErrDisallowed = errors.New("disallowed module version") // CheckExclusions returns an error equivalent to ErrDisallowed if module m is // excluded by the main module's go.mod file. func CheckExclusions(ctx context.Context, m module.Version) error { - if index != nil && index.exclude[m] { - return module.VersionError(m, errExcluded) + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } } return nil } @@ -306,23 +323,77 @@ func CheckDeprecation(ctx context.Context, m module.Version) (deprecation string return summary.deprecated, nil } -// Replacement returns the replacement for mod, if any, from go.mod. -// If there is no replacement for mod, Replacement returns -// a module.Version with Path == "". +func replacement(mod module.Version, replace map[module.Version]module.Version) (fromVersion string, to module.Version, ok bool) { + if r, ok := replace[mod]; ok { + return mod.Version, r, true + } + if r, ok := replace[module.Version{Path: mod.Path}]; ok { + return "", r, true + } + return "", module.Version{}, false +} + +// Replacement returns the replacement for mod, if any. If the path in the +// module.Version is relative it's relative to the single main module outside +// workspace mode, or the workspace's directory in workspace mode. func Replacement(mod module.Version) module.Version { - if index != nil { - if r, ok := index.replace[mod]; ok { - return r - } - if r, ok := index.replace[module.Version{Path: mod.Path}]; ok { - return r + foundFrom, found, foundModRoot := "", module.Version{}, "" + if MainModules == nil { + return module.Version{} + } else if MainModules.Contains(mod.Path) && mod.Version == "" { + // Don't replace the workspace version of the main module. + return module.Version{} + } + if _, r, ok := replacement(mod, MainModules.WorkFileReplaceMap()); ok { + return r + } + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil { + if from, r, ok := replacement(mod, index.replace); ok { + modRoot := MainModules.ModRoot(v) + if foundModRoot != "" && foundFrom != from && found != r { + base.Errorf("conflicting replacements found for %v in workspace modules defined by %v and %v", + mod, modFilePath(foundModRoot), modFilePath(modRoot)) + return canonicalizeReplacePath(found, foundModRoot) + } + found, foundModRoot = r, modRoot + } } } - return module.Version{} + return canonicalizeReplacePath(found, foundModRoot) +} + +func replaceRelativeTo() string { + if workFilePath := WorkFilePath(); workFilePath != "" { + return filepath.Dir(workFilePath) + } + return MainModules.ModRoot(MainModules.mustGetSingleMainModule()) +} + +// canonicalizeReplacePath ensures that relative, on-disk, replaced module paths +// are relative to the workspace directory (in workspace mode) or to the module's +// directory (in module mode, as they already are). +func canonicalizeReplacePath(r module.Version, modRoot string) module.Version { + if filepath.IsAbs(r.Path) || r.Version != "" { + return r + } + workFilePath := WorkFilePath() + if workFilePath == "" { + return r + } + abs := filepath.Join(modRoot, r.Path) + if rel, err := filepath.Rel(filepath.Dir(workFilePath), abs); err == nil { + return module.Version{Path: rel, Version: r.Version} + } + // We couldn't make the version's path relative to the workspace's path, + // so just return the absolute path. It's the best we can do. + return module.Version{Path: abs, Version: r.Version} } // resolveReplacement returns the module actually used to load the source code // for m: either m itself, or the replacement for m (iff m is replaced). +// It also returns the modroot of the module providing the replacement if +// one was found. func resolveReplacement(m module.Version) module.Version { if r := Replacement(m); r.Path != "" { return r @@ -330,10 +401,21 @@ func resolveReplacement(m module.Version) module.Version { return m } +func toReplaceMap(replacements []*modfile.Replace) map[module.Version]module.Version { + replaceMap := make(map[module.Version]module.Version, len(replacements)) + for _, r := range replacements { + if prev, dup := replaceMap[r.Old]; dup && prev != r.New { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New) + } + replaceMap[r.Old] = r.New + } + return replaceMap +} + // indexModFile rebuilds the index of modFile. // If modFile has been changed since it was first read, // modFile.Cleanup must be called before indexModFile. -func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileIndex { +func indexModFile(data []byte, modFile *modfile.File, mod module.Version, needsFix bool) *modFileIndex { i := new(modFileIndex) i.data = data i.dataNeedsFix = needsFix @@ -345,12 +427,12 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.goVersionV = "" if modFile.Go == nil { - rawGoVersion.Store(Target, "") + rawGoVersion.Store(mod, "") } else { // We're going to use the semver package to compare Go versions, so go ahead // and add the "v" prefix it expects once instead of every time. i.goVersionV = "v" + modFile.Go.Version - rawGoVersion.Store(Target, modFile.Go.Version) + rawGoVersion.Store(mod, modFile.Go.Version) } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -358,21 +440,7 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.require[r.Mod] = requireMeta{indirect: r.Indirect} } - i.replace = make(map[module.Version]module.Version, len(modFile.Replace)) - for _, r := range modFile.Replace { - if prev, dup := i.replace[r.Old]; dup && prev != r.New { - base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New) - } - i.replace[r.Old] = r.New - } - - i.highestReplaced = make(map[string]string) - for _, r := range modFile.Replace { - v, ok := i.highestReplaced[r.Old.Path] - if !ok || semver.Compare(r.Old.Version, v) > 0 { - i.highestReplaced[r.Old.Path] = r.Old.Version - } - } + i.replace = toReplaceMap(modFile.Replace) i.exclude = make(map[module.Version]bool, len(modFile.Exclude)) for _, x := range modFile.Exclude { @@ -465,7 +533,7 @@ var rawGoVersion sync.Map // map[module.Version]string type modFileSummary struct { module module.Version goVersion string - depth modDepth + pruning modPruning require []module.Version retract []retraction deprecated string @@ -490,8 +558,8 @@ type retraction struct { // // The caller must not modify the returned summary. func goModSummary(m module.Version) (*modFileSummary, error) { - if m == Target { - panic("internal error: goModSummary called on the Target module") + if m.Version == "" && !inWorkspaceMode() && MainModules.Contains(m.Path) { + panic("internal error: goModSummary called on a main module") } if cfg.BuildMod == "vendor" { @@ -506,7 +574,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { // For every module other than the target, // return the full list of modules from modules.txt. - readVendorList() + readVendorList(MainModules.mustGetSingleMainModule()) // We don't know what versions the vendored module actually relies on, // so assume that it requires everything. @@ -515,7 +583,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { } actual := resolveReplacement(m) - if HasModRoot() && cfg.BuildMod == "readonly" && actual.Version != "" { + if HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() && actual.Version != "" { key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} if !modfetch.HaveSum(key) { suggestion := fmt.Sprintf("; to add it:\n\tgo mod download %s", m.Path) @@ -553,27 +621,29 @@ func goModSummary(m module.Version) (*modFileSummary, error) { } } - if index != nil && len(index.exclude) > 0 { - // Drop any requirements on excluded versions. - // Don't modify the cached summary though, since we might need the raw - // summary separately. - haveExcludedReqs := false - for _, r := range summary.require { - if index.exclude[r] { - haveExcludedReqs = true - break - } - } - if haveExcludedReqs { - s := new(modFileSummary) - *s = *summary - s.require = make([]module.Version, 0, len(summary.require)) + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false for _, r := range summary.require { - if !index.exclude[r] { - s.require = append(s.require, r) + if index.exclude[r] { + haveExcludedReqs = true + break } } - summary = s + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } } } return summary, nil @@ -584,16 +654,20 @@ func goModSummary(m module.Version) (*modFileSummary, error) { // its dependencies. // // rawGoModSummary cannot be used on the Target module. + func rawGoModSummary(m module.Version) (*modFileSummary, error) { - if m == Target { + if m.Path == "" && MainModules.Contains(m.Path) { panic("internal error: rawGoModSummary called on the Target module") } + type key struct { + m module.Version + } type cached struct { summary *modFileSummary err error } - c := rawGoModSummaryCache.Do(m, func() interface{} { + c := rawGoModSummaryCache.Do(key{m}, func() any { summary := new(modFileSummary) name, data, err := rawGoModData(m) if err != nil { @@ -610,9 +684,9 @@ func rawGoModSummary(m module.Version) (*modFileSummary, error) { if f.Go != nil && f.Go.Version != "" { rawGoVersion.LoadOrStore(m, f.Go.Version) summary.goVersion = f.Go.Version - summary.depth = modDepthFromGoVersion(f.Go.Version) + summary.pruning = pruningForGoVersion(f.Go.Version) } else { - summary.depth = eager + summary.pruning = unpruned } if len(f.Require) > 0 { summary.require = make([]module.Version, 0, len(f.Require)) @@ -648,9 +722,14 @@ var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result func rawGoModData(m module.Version) (name string, data []byte, err error) { if m.Version == "" { // m is a replacement module with only a file path. + dir := m.Path if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) + if inWorkspaceMode() && MainModules.Contains(m.Path) { + dir = MainModules.ModRoot(m) + } else { + dir = filepath.Join(replaceRelativeTo(), dir) + } } name = filepath.Join(dir, "go.mod") if gomodActual, ok := fsys.OverlayPath(name); ok { @@ -690,7 +769,7 @@ func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (la latest module.Version err error } - e := latestVersionIgnoringRetractionsCache.Do(path, func() interface{} { + e := latestVersionIgnoringRetractionsCache.Do(path, func() any { ctx, span := trace.StartSpan(ctx, "queryLatestVersionIgnoringRetractions "+path) defer span.Done() @@ -718,3 +797,15 @@ func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (la } var latestVersionIgnoringRetractionsCache par.Cache // path → queryLatestVersionIgnoringRetractions result + +// ToDirectoryPath adds a prefix if necessary so that path in unambiguously +// an absolute path or a relative path starting with a '.' or '..' +// path component. +func ToDirectoryPath(path string) string { + if modfile.IsDirectoryPath(path) { + return path + } + // The path is not a relative path or an absolute path, so make it relative + // to the current directory. + return "./" + filepath.ToSlash(filepath.Clean(path)) +} diff --git a/libgo/go/cmd/go/internal/modload/mvs.go b/libgo/go/cmd/go/internal/modload/mvs.go index 87619b4ace6..588bcf4bdc2 100644 --- a/libgo/go/cmd/go/internal/modload/mvs.go +++ b/libgo/go/cmd/go/internal/modload/mvs.go @@ -42,7 +42,7 @@ type mvsReqs struct { } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - if mod == Target { + if mod.Version == "" && MainModules.Contains(mod.Path) { // Use the build list as it existed when r was constructed, not the current // global build list. return r.roots, nil @@ -108,12 +108,12 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, // previousVersion returns the tagged version of m.Path immediately prior to // m.Version, or version "none" if no prior version is tagged. // -// Since the version of Target is not found in the version list, +// Since the version of a main module is not found in the version list, // it has no previous version. func previousVersion(m module.Version) (module.Version, error) { // TODO(golang.org/issue/38714): thread tracing context through MVS. - if m == Target { + if m.Version == "" && MainModules.Contains(m.Path) { return module.Version{Path: m.Path, Version: "none"}, nil } diff --git a/libgo/go/cmd/go/internal/modload/query.go b/libgo/go/cmd/go/internal/modload/query.go index e737ca90fcd..33808ea1097 100644 --- a/libgo/go/cmd/go/internal/modload/query.go +++ b/libgo/go/cmd/go/internal/modload/query.go @@ -110,11 +110,12 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed allowed = func(context.Context, module.Version) error { return nil } } - if path == Target.Path && (query == "upgrade" || query == "patch") { - if err := allowed(ctx, Target); err != nil { + if MainModules.Contains(path) && (query == "upgrade" || query == "patch") { + m := module.Version{Path: path} + if err := allowed(ctx, m); err != nil { return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } - return &modfetch.RevInfo{Version: Target.Version}, nil + return &modfetch.RevInfo{Version: m.Version}, nil } if path == "std" || path == "cmd" { @@ -512,9 +513,10 @@ func QueryPackages(ctx context.Context, pattern, query string, current func(stri pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) if len(pkgMods) == 0 && err == nil { + replacement := Replacement(modOnly.Mod) return nil, &PackageNotInModuleError{ Mod: modOnly.Mod, - Replacement: Replacement(modOnly.Mod), + Replacement: replacement, Query: query, Pattern: pattern, } @@ -551,7 +553,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return m.Errs[0] } - var match func(mod module.Version, root string, isLocal bool) *search.Match + var match func(mod module.Version, roots []string, isLocal bool) *search.Match matchPattern := search.MatchPattern(pattern) if i := strings.Index(pattern, "..."); i >= 0 { @@ -559,30 +561,32 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if base == "." { return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} } - match = func(mod module.Version, root string, isLocal bool) *search.Match { + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { m := search.NewMatch(pattern) matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { - match = func(mod module.Version, root string, isLocal bool) *search.Match { + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { m := search.NewMatch(pattern) prefix := mod.Path - if mod == Target { - prefix = targetPrefix + if MainModules.Contains(mod.Path) { + prefix = MainModules.PathPrefix(module.Version{Path: mod.Path}) } - if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { - m.AddError(err) - } else if ok { - m.Pkgs = []string{pattern} + for _, root := range roots { + if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { + m.AddError(err) + } else if ok { + m.Pkgs = []string{pattern} + } } return m } } - var queryMatchesMainModule bool - if HasModRoot() { - m := match(Target, modRoot, true) + var mainModuleMatches []module.Version + for _, mainModule := range MainModules.Versions() { + m := match(mainModule, modRoots, true) if len(m.Pkgs) > 0 { if query != "upgrade" && query != "patch" { return nil, nil, &QueryMatchesPackagesInMainModuleError{ @@ -591,12 +595,12 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin Packages: m.Pkgs, } } - if err := allowed(ctx, Target); err != nil { - return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) + if err := allowed(ctx, mainModule); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, mainModule.Path, err) } return []QueryResult{{ - Mod: Target, - Rev: &modfetch.RevInfo{Version: Target.Version}, + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, Packages: m.Pkgs, }}, nil, nil } @@ -604,15 +608,17 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return nil, nil, err } - if matchPattern(Target.Path) { - queryMatchesMainModule = true + var matchesMainModule bool + if matchPattern(mainModule.Path) { + mainModuleMatches = append(mainModuleMatches, mainModule) + matchesMainModule = true } - if (query == "upgrade" || query == "patch") && queryMatchesMainModule { - if err := allowed(ctx, Target); err == nil { + if (query == "upgrade" || query == "patch") && matchesMainModule { + if err := allowed(ctx, mainModule); err == nil { modOnly = &QueryResult{ - Mod: Target, - Rev: &modfetch.RevInfo{Version: Target.Version}, + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, } } } @@ -625,16 +631,17 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if len(candidateModules) == 0 { if modOnly != nil { return nil, modOnly, nil - } else if queryMatchesMainModule { - return nil, nil, &QueryMatchesMainModuleError{ - Pattern: pattern, - Query: query, + } else if len(mainModuleMatches) != 0 { + return nil, nil, &QueryMatchesMainModulesError{ + MainModules: mainModuleMatches, + Pattern: pattern, + Query: query, } } else { return nil, nil, &PackageNotInModuleError{ - Mod: Target, - Query: query, - Pattern: pattern, + MainModules: mainModuleMatches, + Query: query, + Pattern: pattern, } } } @@ -656,15 +663,16 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if err != nil { return r, err } - m := match(r.Mod, root, isLocal) + m := match(r.Mod, []string{root}, isLocal) r.Packages = m.Pkgs if len(r.Packages) == 0 && !matchPattern(path) { if err := firstError(m); err != nil { return r, err } + replacement := Replacement(r.Mod) return r, &PackageNotInModuleError{ Mod: r.Mod, - Replacement: Replacement(r.Mod), + Replacement: replacement, Query: query, Pattern: pattern, } @@ -684,8 +692,8 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return err }) - if queryMatchesMainModule && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { - return nil, nil, &QueryMatchesMainModuleError{ + if len(mainModuleMatches) > 0 && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + return nil, nil, &QueryMatchesMainModulesError{ Pattern: pattern, Query: query, } @@ -701,8 +709,13 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin func modulePrefixesExcludingTarget(path string) []string { prefixes := make([]string, 0, strings.Count(path, "/")+1) + mainModulePrefixes := make(map[string]bool) + for _, m := range MainModules.Versions() { + mainModulePrefixes[m.Path] = true + } + for { - if path != targetPrefix { + if !mainModulePrefixes[path] { if _, _, ok := module.SplitPathVersion(path); ok { prefixes = append(prefixes, path) } @@ -759,7 +772,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod case *PackageNotInModuleError: // Given the option, prefer to attribute “package not in module” // to modules other than the main one. - if noPackage == nil || noPackage.Mod == Target { + if noPackage == nil || MainModules.Contains(noPackage.Mod.Path) { noPackage = rErr } case *NoMatchingVersionError: @@ -878,6 +891,7 @@ func (e *WildcardInFirstElementError) Error() string { // code for the versions it knows about, and thus did not have the opportunity // to return a non-400 status code to suppress fallback. type PackageNotInModuleError struct { + MainModules []module.Version Mod module.Version Replacement module.Version Query string @@ -885,11 +899,15 @@ type PackageNotInModuleError struct { } func (e *PackageNotInModuleError) Error() string { - if e.Mod == Target { + if len(e.MainModules) > 0 { + prefix := "workspace modules do" + if len(e.MainModules) == 1 { + prefix = fmt.Sprintf("main module (%s) does", e.MainModules[0]) + } if strings.Contains(e.Pattern, "...") { - return fmt.Sprintf("main module (%s) does not contain packages matching %s", Target.Path, e.Pattern) + return fmt.Sprintf("%s not contain packages matching %s", prefix, e.Pattern) } - return fmt.Sprintf("main module (%s) does not contain package %s", Target.Path, e.Pattern) + return fmt.Sprintf("%s not contain package %s", prefix, e.Pattern) } found := "" @@ -978,14 +996,13 @@ func lookupRepo(proxy, path string) (repo versionRepo, err error) { repo = emptyRepo{path: path, err: err} } - if index == nil { - return repo, err - } - if _, ok := index.highestReplaced[path]; !ok { + if MainModules == nil { return repo, err + } else if _, ok := MainModules.HighestReplaced()[path]; ok { + return &replacementRepo{repo: repo}, nil } - return &replacementRepo{repo: repo}, nil + return repo, err } // An emptyRepo is a versionRepo that contains no versions. @@ -1024,11 +1041,13 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) { } versions := repoVersions - if index != nil && len(index.replace) > 0 { - path := rr.ModulePath() - for m, _ := range index.replace { - if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !module.IsPseudoVersion(m.Version) { - versions = append(versions, m.Version) + for _, mm := range MainModules.Versions() { + if index := MainModules.Index(mm); index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !module.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } } } } @@ -1046,7 +1065,16 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) { func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { info, err := rr.repo.Stat(rev) - if err == nil || index == nil || len(index.replace) == 0 { + if err == nil { + return info, err + } + var hasReplacements bool + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil && len(index.replace) > 0 { + hasReplacements = true + } + } + if !hasReplacements { return info, err } @@ -1073,26 +1101,24 @@ func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { info, err := rr.repo.Latest() + path := rr.ModulePath() - if index != nil { - path := rr.ModulePath() - if v, ok := index.highestReplaced[path]; ok { - if v == "" { - // The only replacement is a wildcard that doesn't specify a version, so - // synthesize a pseudo-version with an appropriate major version and a - // timestamp below any real timestamp. That way, if the main module is - // used from within some other module, the user will be able to upgrade - // the requirement to any real version they choose. - if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { - v = module.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") - } else { - v = module.PseudoVersion("v0", "", time.Time{}, "000000000000") - } + if v, ok := MainModules.HighestReplaced()[path]; ok { + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = module.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = module.PseudoVersion("v0", "", time.Time{}, "000000000000") } + } - if err != nil || semver.Compare(v, info.Version) > 0 { - return rr.replacementStat(v) - } + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) } } @@ -1108,20 +1134,46 @@ func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) return rev, nil } -// A QueryMatchesMainModuleError indicates that a query requests +// A QueryMatchesMainModulesError indicates that a query requests // a version of the main module that cannot be satisfied. // (The main module's version cannot be changed.) -type QueryMatchesMainModuleError struct { - Pattern string - Query string +type QueryMatchesMainModulesError struct { + MainModules []module.Version + Pattern string + Query string } -func (e *QueryMatchesMainModuleError) Error() string { - if e.Pattern == Target.Path { +func (e *QueryMatchesMainModulesError) Error() string { + if MainModules.Contains(e.Pattern) { return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) } - return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, Target.Path) + plural := "" + mainModulePaths := make([]string, len(e.MainModules)) + for i := range e.MainModules { + mainModulePaths[i] = e.MainModules[i].Path + } + if len(e.MainModules) > 1 { + plural = "s" + } + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module%s (%s)", e.Query, e.Pattern, plural, strings.Join(mainModulePaths, ", ")) +} + +// A QueryUpgradesAllError indicates that a query requests +// an upgrade on the all pattern. +// (The main module's version cannot be changed.) +type QueryUpgradesAllError struct { + MainModules []module.Version + Query string +} + +func (e *QueryUpgradesAllError) Error() string { + var plural string = "" + if len(e.MainModules) != 1 { + plural = "s" + } + + return fmt.Sprintf("can't request version %q of pattern \"all\" that includes the main module%s", e.Query, plural) } // A QueryMatchesPackagesInMainModuleError indicates that a query cannot be diff --git a/libgo/go/cmd/go/internal/modload/search.go b/libgo/go/cmd/go/internal/modload/search.go index 0c2e2660e9f..87c5c55471d 100644 --- a/libgo/go/cmd/go/internal/modload/search.go +++ b/libgo/go/cmd/go/internal/modload/search.go @@ -132,9 +132,10 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f } if cfg.BuildMod == "vendor" { - if HasModRoot() { - walkPkgs(ModRoot(), targetPrefix, pruneGoMod|pruneVendor) - walkPkgs(filepath.Join(ModRoot(), "vendor"), "", pruneVendor) + mod := MainModules.mustGetSingleMainModule() + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor) + walkPkgs(filepath.Join(modRoot, "vendor"), "", pruneVendor) } return } @@ -148,12 +149,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f root, modPrefix string isLocal bool ) - if mod == Target { - if !HasModRoot() { + if MainModules.Contains(mod.Path) { + if MainModules.ModRoot(mod) == "" { continue // If there is no main module, we can't search in it. } - root = ModRoot() - modPrefix = targetPrefix + root = MainModules.ModRoot(mod) + modPrefix = MainModules.PathPrefix(mod) isLocal = true } else { var err error diff --git a/libgo/go/cmd/go/internal/modload/stat_openfile.go b/libgo/go/cmd/go/internal/modload/stat_openfile.go index 368f8931984..ff7c124af58 100644 --- a/libgo/go/cmd/go/internal/modload/stat_openfile.go +++ b/libgo/go/cmd/go/internal/modload/stat_openfile.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build (js && wasm) || plan9 -// +build js,wasm plan9 // On plan9, per http://9p.io/magic/man2html/2/access: “Since file permissions // are checked by the server and group information is not known to the client, diff --git a/libgo/go/cmd/go/internal/modload/stat_unix.go b/libgo/go/cmd/go/internal/modload/stat_unix.go index f5f7ce3e03d..38f5cc4ffb6 100644 --- a/libgo/go/cmd/go/internal/modload/stat_unix.go +++ b/libgo/go/cmd/go/internal/modload/stat_unix.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build aix || darwin || dragonfly || freebsd || hurd || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris package modload diff --git a/libgo/go/cmd/go/internal/modload/stat_windows.go b/libgo/go/cmd/go/internal/modload/stat_windows.go index 825e60b27af..f29a99165e5 100644 --- a/libgo/go/cmd/go/internal/modload/stat_windows.go +++ b/libgo/go/cmd/go/internal/modload/stat_windows.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build windows -// +build windows package modload diff --git a/libgo/go/cmd/go/internal/modload/vendor.go b/libgo/go/cmd/go/internal/modload/vendor.go index 80713b0812e..5ea82a86208 100644 --- a/libgo/go/cmd/go/internal/modload/vendor.go +++ b/libgo/go/cmd/go/internal/modload/vendor.go @@ -15,6 +15,7 @@ import ( "cmd/go/internal/base" + "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -35,13 +36,13 @@ type vendorMetadata struct { } // readVendorList reads the list of vendored modules from vendor/modules.txt. -func readVendorList() { +func readVendorList(mainModule module.Version) { vendorOnce.Do(func() { vendorList = nil vendorPkgModule = make(map[string]module.Version) vendorVersion = make(map[string]string) vendorMeta = make(map[module.Version]vendorMetadata) - data, err := os.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + data, err := os.ReadFile(filepath.Join(MainModules.ModRoot(mainModule), "vendor/modules.txt")) if err != nil { if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) @@ -134,8 +135,8 @@ func readVendorList() { // checkVendorConsistency verifies that the vendor/modules.txt file matches (if // go 1.14) or at least does not contradict (go 1.13 or earlier) the // requirements and replacements listed in the main module's go.mod file. -func checkVendorConsistency() { - readVendorList() +func checkVendorConsistency(index *modFileIndex, modFile *modfile.File) { + readVendorList(MainModules.mustGetSingleMainModule()) pre114 := false if semver.Compare(index.goVersionV, "v1.14") < 0 { @@ -146,7 +147,7 @@ func checkVendorConsistency() { } vendErrors := new(strings.Builder) - vendErrorf := func(mod module.Version, format string, args ...interface{}) { + vendErrorf := func(mod module.Version, format string, args ...any) { detail := fmt.Sprintf(format, args...) if mod.Version == "" { fmt.Fprintf(vendErrors, "\n\t%s: %s", mod.Path, detail) @@ -219,6 +220,7 @@ func checkVendorConsistency() { } if vendErrors.Len() > 0 { + modRoot := MainModules.ModRoot(MainModules.mustGetSingleMainModule()) base.Fatalf("go: inconsistent vendoring in %s:%s\n\n\tTo ignore the vendor directory, use -mod=readonly or -mod=mod.\n\tTo sync the vendor directory, run:\n\t\tgo mod vendor", modRoot, vendErrors) } } diff --git a/libgo/go/cmd/go/internal/mvs/mvs.go b/libgo/go/cmd/go/internal/mvs/mvs.go index 6969f90f2e6..d25d447b0ee 100644 --- a/libgo/go/cmd/go/internal/mvs/mvs.go +++ b/libgo/go/cmd/go/internal/mvs/mvs.go @@ -8,6 +8,7 @@ package mvs import ( "fmt" + "reflect" "sort" "sync" @@ -85,11 +86,11 @@ type DowngradeReqs interface { // of the list are sorted by path. // // See https://research.swtch.com/vgo-mvs for details. -func BuildList(target module.Version, reqs Reqs) ([]module.Version, error) { - return buildList(target, reqs, nil) +func BuildList(targets []module.Version, reqs Reqs) ([]module.Version, error) { + return buildList(targets, reqs, nil) } -func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { +func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { cmp := func(v1, v2 string) int { if reqs.Max(v1, v2) != v1 { return -1 @@ -102,7 +103,7 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m var ( mu sync.Mutex - g = NewGraph(cmp, []module.Version{target}) + g = NewGraph(cmp, targets) upgrades = map[module.Version]module.Version{} errs = map[module.Version]error{} // (non-nil errors only) ) @@ -110,8 +111,10 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // Explore work graph in parallel in case reqs.Required // does high-latency network operations. var work par.Work - work.Add(target) - work.Do(10, func(item interface{}) { + for _, target := range targets { + work.Add(target) + } + work.Do(10, func(item any) { m := item.(module.Version) var required []module.Version @@ -168,12 +171,12 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // The final list is the minimum version of each module found in the graph. list := g.BuildList() - if v := list[0]; v != target { + if vs := list[:len(targets)]; !reflect.DeepEqual(vs, targets) { // target.Version will be "" for modload, the main client of MVS. // "" denotes the main module, which has no version. However, MVS treats // version strings as opaque, so "" is not a special value here. // See golang.org/issue/31491, golang.org/issue/29773. - panic(fmt.Sprintf("mistake: chose version %q instead of target %+v", v, target)) + panic(fmt.Sprintf("mistake: chose versions %+v instead of targets %+v", vs, targets)) } return list, nil } @@ -181,8 +184,8 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // Req returns the minimal requirement list for the target module, // with the constraint that all module paths listed in base must // appear in the returned list. -func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, error) { - list, err := BuildList(target, reqs) +func Req(mainModule module.Version, base []string, reqs Reqs) ([]module.Version, error) { + list, err := BuildList([]module.Version{mainModule}, reqs) if err != nil { return nil, err } @@ -194,7 +197,8 @@ func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, err // Compute postorder, cache requirements. var postorder []module.Version reqCache := map[module.Version][]module.Version{} - reqCache[target] = nil + reqCache[mainModule] = nil + var walk func(module.Version) error walk = func(m module.Version) error { _, ok := reqCache[m] @@ -273,7 +277,7 @@ func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, err // UpgradeAll returns a build list for the target module // in which every module is upgraded to its latest version. func UpgradeAll(target module.Version, reqs UpgradeReqs) ([]module.Version, error) { - return buildList(target, reqs, func(m module.Version) (module.Version, error) { + return buildList([]module.Version{target}, reqs, func(m module.Version) (module.Version, error) { if m.Path == target.Path { return target, nil } @@ -308,7 +312,7 @@ func Upgrade(target module.Version, reqs UpgradeReqs, upgrade ...module.Version) } } - return buildList(target, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + return buildList([]module.Version{target}, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { if v, ok := upgradeTo[m.Path]; ok { return module.Version{Path: m.Path, Version: v}, nil } @@ -331,7 +335,7 @@ func Downgrade(target module.Version, reqs DowngradeReqs, downgrade ...module.Ve // // In order to generate those new requirements, we need to identify versions // for every module in the build list — not just reqs.Required(target). - list, err := BuildList(target, reqs) + list, err := BuildList([]module.Version{target}, reqs) if err != nil { return nil, err } @@ -446,7 +450,7 @@ List: // list with the actual versions of the downgraded modules as selected by MVS, // instead of our initial downgrades. // (See the downhiddenartifact and downhiddencross test cases). - actual, err := BuildList(target, &override{ + actual, err := BuildList([]module.Version{target}, &override{ target: target, list: downgraded, Reqs: reqs, @@ -466,7 +470,7 @@ List: } } - return BuildList(target, &override{ + return BuildList([]module.Version{target}, &override{ target: target, list: downgraded, Reqs: reqs, diff --git a/libgo/go/cmd/go/internal/mvs/mvs_test.go b/libgo/go/cmd/go/internal/mvs/mvs_test.go index 598ed666889..26d004fee28 100644 --- a/libgo/go/cmd/go/internal/mvs/mvs_test.go +++ b/libgo/go/cmd/go/internal/mvs/mvs_test.go @@ -507,7 +507,7 @@ func Test(t *testing.T) { t.Fatalf("build takes one argument: %q", line) } fns = append(fns, func(t *testing.T) { - list, err := BuildList(m(kf[1]), reqs) + list, err := BuildList([]module.Version{m(kf[1])}, reqs) checkList(t, key, list, err, val) }) continue diff --git a/libgo/go/cmd/go/internal/par/work.go b/libgo/go/cmd/go/internal/par/work.go index 960cec6fb16..496c41b1509 100644 --- a/libgo/go/cmd/go/internal/par/work.go +++ b/libgo/go/cmd/go/internal/par/work.go @@ -14,24 +14,24 @@ import ( // Work manages a set of work items to be executed in parallel, at most once each. // The items in the set must all be valid map keys. type Work struct { - f func(interface{}) // function to run for each item - running int // total number of runners + f func(any) // function to run for each item + running int // total number of runners mu sync.Mutex - added map[interface{}]bool // items added to set - todo []interface{} // items yet to be run - wait sync.Cond // wait when todo is empty - waiting int // number of runners waiting for todo + added map[any]bool // items added to set + todo []any // items yet to be run + wait sync.Cond // wait when todo is empty + waiting int // number of runners waiting for todo } func (w *Work) init() { if w.added == nil { - w.added = make(map[interface{}]bool) + w.added = make(map[any]bool) } } // Add adds item to the work set, if it hasn't already been added. -func (w *Work) Add(item interface{}) { +func (w *Work) Add(item any) { w.mu.Lock() w.init() if !w.added[item] { @@ -51,7 +51,7 @@ func (w *Work) Add(item interface{}) { // before calling Do (or else Do returns immediately), // but it is allowed for f(item) to add new items to the set. // Do should only be used once on a given Work. -func (w *Work) Do(n int, f func(item interface{})) { +func (w *Work) Do(n int, f func(item any)) { if n < 1 { panic("par.Work.Do: n < 1") } @@ -110,13 +110,13 @@ type Cache struct { type cacheEntry struct { done uint32 mu sync.Mutex - result interface{} + result any } // Do calls the function f if and only if Do is being called for the first time with this key. // No call to Do with a given key returns until the one call to f returns. // Do returns the value returned by the one call to f. -func (c *Cache) Do(key interface{}, f func() interface{}) interface{} { +func (c *Cache) Do(key any, f func() any) any { entryIface, ok := c.m.Load(key) if !ok { entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry)) @@ -136,7 +136,7 @@ func (c *Cache) Do(key interface{}, f func() interface{}) interface{} { // Get returns the cached result associated with key. // It returns nil if there is no such result. // If the result for key is being computed, Get does not wait for the computation to finish. -func (c *Cache) Get(key interface{}) interface{} { +func (c *Cache) Get(key any) any { entryIface, ok := c.m.Load(key) if !ok { return nil @@ -156,7 +156,7 @@ func (c *Cache) Get(key interface{}) interface{} { // TODO(jayconrod): Delete this after the package cache clearing functions // in internal/load have been removed. func (c *Cache) Clear() { - c.m.Range(func(key, value interface{}) bool { + c.m.Range(func(key, value any) bool { c.m.Delete(key) return true }) @@ -169,7 +169,7 @@ func (c *Cache) Clear() { // // TODO(jayconrod): Delete this after the package cache clearing functions // in internal/load have been removed. -func (c *Cache) Delete(key interface{}) { +func (c *Cache) Delete(key any) { c.m.Delete(key) } @@ -180,8 +180,8 @@ func (c *Cache) Delete(key interface{}) { // // TODO(jayconrod): Delete this after the package cache clearing functions // in internal/load have been removed. -func (c *Cache) DeleteIf(pred func(key interface{}) bool) { - c.m.Range(func(key, _ interface{}) bool { +func (c *Cache) DeleteIf(pred func(key any) bool) { + c.m.Range(func(key, _ any) bool { if pred(key) { c.Delete(key) } diff --git a/libgo/go/cmd/go/internal/par/work_test.go b/libgo/go/cmd/go/internal/par/work_test.go index f104bc4106f..add0e640d8c 100644 --- a/libgo/go/cmd/go/internal/par/work_test.go +++ b/libgo/go/cmd/go/internal/par/work_test.go @@ -16,7 +16,7 @@ func TestWork(t *testing.T) { const N = 10000 n := int32(0) w.Add(N) - w.Do(100, func(x interface{}) { + w.Do(100, func(x any) { atomic.AddInt32(&n, 1) i := x.(int) if i >= 2 { @@ -40,7 +40,7 @@ func TestWorkParallel(t *testing.T) { } start := time.Now() var n int32 - w.Do(N, func(x interface{}) { + w.Do(N, func(x any) { time.Sleep(1 * time.Millisecond) atomic.AddInt32(&n, +1) }) @@ -58,19 +58,19 @@ func TestCache(t *testing.T) { var cache Cache n := 1 - v := cache.Do(1, func() interface{} { n++; return n }) + v := cache.Do(1, func() any { n++; return n }) if v != 2 { t.Fatalf("cache.Do(1) did not run f") } - v = cache.Do(1, func() interface{} { n++; return n }) + v = cache.Do(1, func() any { n++; return n }) if v != 2 { t.Fatalf("cache.Do(1) ran f again!") } - v = cache.Do(2, func() interface{} { n++; return n }) + v = cache.Do(2, func() any { n++; return n }) if v != 3 { t.Fatalf("cache.Do(2) did not run f") } - v = cache.Do(1, func() interface{} { n++; return n }) + v = cache.Do(1, func() any { n++; return n }) if v != 2 { t.Fatalf("cache.Do(1) did not returned saved value from original cache.Do(1)") } diff --git a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go index d5c241857b4..c56e36ca624 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go +++ b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build windows || darwin -// +build windows darwin package robustio diff --git a/libgo/go/cmd/go/internal/robustio/robustio_other.go b/libgo/go/cmd/go/internal/robustio/robustio_other.go index 3a20cac6cf8..da9a46e4fac 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio_other.go +++ b/libgo/go/cmd/go/internal/robustio/robustio_other.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !windows && !darwin -// +build !windows,!darwin package robustio diff --git a/libgo/go/cmd/go/internal/run/run.go b/libgo/go/cmd/go/internal/run/run.go index 784f7162dfd..c4b70b64fe6 100644 --- a/libgo/go/cmd/go/internal/run/run.go +++ b/libgo/go/cmd/go/internal/run/run.go @@ -65,14 +65,17 @@ func init() { CmdRun.Run = runRun // break init loop work.AddBuildFlags(CmdRun, work.DefaultBuildFlags) + base.AddWorkfileFlag(&CmdRun.Flag) CmdRun.Flag.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "") } -func printStderr(args ...interface{}) (int, error) { +func printStderr(args ...any) (int, error) { return fmt.Fprint(os.Stderr, args...) } func runRun(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if shouldUseOutsideModuleMode(args) { // Set global module flags for 'go run cmd@version'. // This must be done before modload.Init, but we need to call work.BuildInit @@ -100,7 +103,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { if strings.HasSuffix(file, "_test.go") { // GoFilesPackage is going to assign this to TestGoFiles. // Reject since it won't be part of the build. - base.Fatalf("go run: cannot run *_test.go files (%s)", file) + base.Fatalf("go: cannot run *_test.go files (%s)", file) } } p = load.GoFilesPackage(ctx, pkgOpts, files) @@ -111,26 +114,26 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { var err error pkgs, err = load.PackagesAndErrorsOutsideModule(ctx, pkgOpts, args[:1]) if err != nil { - base.Fatalf("go run: %v", err) + base.Fatalf("go: %v", err) } } else { pkgs = load.PackagesAndErrors(ctx, pkgOpts, args[:1]) } if len(pkgs) == 0 { - base.Fatalf("go run: no packages loaded from %s", arg) + base.Fatalf("go: no packages loaded from %s", arg) } if len(pkgs) > 1 { var names []string for _, p := range pkgs { names = append(names, p.ImportPath) } - base.Fatalf("go run: pattern %s matches multiple packages:\n\t%s", arg, strings.Join(names, "\n\t")) + base.Fatalf("go: pattern %s matches multiple packages:\n\t%s", arg, strings.Join(names, "\n\t")) } p = pkgs[0] i++ } else { - base.Fatalf("go run: no go files listed") + base.Fatalf("go: no go files listed") } cmdArgs := args[i:] load.CheckPackageErrors([]*load.Package{p}) @@ -151,7 +154,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { if !cfg.BuildContext.CgoEnabled { hint = " (cgo is disabled)" } - base.Fatalf("go run: no suitable source files%s", hint) + base.Fatalf("go: no suitable source files%s", hint) } p.Internal.ExeName = src[:len(src)-len(".go")] } else { diff --git a/libgo/go/cmd/go/internal/search/search.go b/libgo/go/cmd/go/internal/search/search.go index a0c806a2593..ebd4990a684 100644 --- a/libgo/go/cmd/go/internal/search/search.go +++ b/libgo/go/cmd/go/internal/search/search.go @@ -202,12 +202,6 @@ func (m *Match) MatchPackages() { } } -var modRoot string - -func SetModRoot(dir string) { - modRoot = dir -} - // MatchDirs sets m.Dirs to a non-nil slice containing all directories that // potentially match a local pattern. The pattern must begin with an absolute // path, or "./", or "../". On Windows, the pattern may use slash or backslash @@ -215,7 +209,7 @@ func SetModRoot(dir string) { // // If any errors may have caused the set of directories to be incomplete, // MatchDirs appends those errors to m.Errs. -func (m *Match) MatchDirs() { +func (m *Match) MatchDirs(modRoots []string) { m.Dirs = []string{} if !m.IsLocal() { m.AddError(fmt.Errorf("internal error: MatchDirs: %s is not a valid filesystem pattern", m.pattern)) @@ -253,15 +247,24 @@ func (m *Match) MatchDirs() { // We need to preserve the ./ for pattern matching // and in the returned import paths. - if modRoot != "" { + if len(modRoots) > 1 { abs, err := filepath.Abs(dir) if err != nil { m.AddError(err) return } - if !hasFilepathPrefix(abs, modRoot) { - m.AddError(fmt.Errorf("directory %s is outside module root (%s)", abs, modRoot)) - return + var found bool + for _, modRoot := range modRoots { + if modRoot != "" && hasFilepathPrefix(abs, modRoot) { + found = true + } + } + if !found { + plural := "" + if len(modRoots) > 1 { + plural = "s" + } + m.AddError(fmt.Errorf("directory %s is outside module root%s (%s)", abs, plural, strings.Join(modRoots, ", "))) } } @@ -424,19 +427,19 @@ func WarnUnmatched(matches []*Match) { // ImportPaths returns the matching paths to use for the given command line. // It calls ImportPathsQuiet and then WarnUnmatched. -func ImportPaths(patterns []string) []*Match { - matches := ImportPathsQuiet(patterns) +func ImportPaths(patterns, modRoots []string) []*Match { + matches := ImportPathsQuiet(patterns, modRoots) WarnUnmatched(matches) return matches } // ImportPathsQuiet is like ImportPaths but does not warn about patterns with no matches. -func ImportPathsQuiet(patterns []string) []*Match { +func ImportPathsQuiet(patterns, modRoots []string) []*Match { var out []*Match for _, a := range CleanPatterns(patterns) { m := NewMatch(a) if m.IsLocal() { - m.MatchDirs() + m.MatchDirs(modRoots) // Change the file import path to a regular import path if the package // is in GOPATH or GOROOT. We don't report errors here; LoadImport diff --git a/libgo/go/cmd/go/internal/str/path.go b/libgo/go/cmd/go/internal/str/path.go index 51ab2af82b5..0c8aaeaca1f 100644 --- a/libgo/go/cmd/go/internal/str/path.go +++ b/libgo/go/cmd/go/internal/str/path.go @@ -49,3 +49,17 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } + +// TrimFilePathPrefix returns s without the leading path elements in prefix. +// If s does not start with prefix (HasFilePathPrefix with the same arguments +// returns false), TrimFilePathPrefix returns s. If s equals prefix, +// TrimFilePathPrefix returns "". +func TrimFilePathPrefix(s, prefix string) string { + if !HasFilePathPrefix(s, prefix) { + return s + } + if len(s) == len(prefix) { + return "" + } + return s[len(prefix)+1:] +} diff --git a/libgo/go/cmd/go/internal/str/str.go b/libgo/go/cmd/go/internal/str/str.go index 9106ebf74d5..021bfbff779 100644 --- a/libgo/go/cmd/go/internal/str/str.go +++ b/libgo/go/cmd/go/internal/str/str.go @@ -14,7 +14,7 @@ import ( // StringList flattens its arguments into a single []string. // Each argument in args must have type string or []string. -func StringList(args ...interface{}) []string { +func StringList(args ...any) []string { var x []string for _, arg := range args { switch arg := arg.(type) { @@ -109,47 +109,3 @@ func Uniq(ss *[]string) { } *ss = uniq } - -func isSpaceByte(c byte) bool { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' -} - -// SplitQuotedFields splits s into a list of fields, -// allowing single or double quotes around elements. -// There is no unescaping or other processing within -// quoted fields. -func SplitQuotedFields(s string) ([]string, error) { - // Split fields allowing '' or "" around elements. - // Quotes further inside the string do not count. - var f []string - for len(s) > 0 { - for len(s) > 0 && isSpaceByte(s[0]) { - s = s[1:] - } - if len(s) == 0 { - break - } - // Accepted quoted string. No unescaping inside. - if s[0] == '"' || s[0] == '\'' { - quote := s[0] - s = s[1:] - i := 0 - for i < len(s) && s[i] != quote { - i++ - } - if i >= len(s) { - return nil, fmt.Errorf("unterminated %c string", quote) - } - f = append(f, s[:i]) - s = s[i+1:] - continue - } - i := 0 - for i < len(s) && !isSpaceByte(s[i]) { - i++ - } - f = append(f, s[:i]) - s = s[i:] - } - return f, nil -} diff --git a/libgo/go/cmd/go/internal/str/str_test.go b/libgo/go/cmd/go/internal/str/str_test.go index 147ce1a63ef..8ea758e0a8b 100644 --- a/libgo/go/cmd/go/internal/str/str_test.go +++ b/libgo/go/cmd/go/internal/str/str_test.go @@ -4,7 +4,9 @@ package str -import "testing" +import ( + "testing" +) var foldDupTests = []struct { list []string diff --git a/libgo/go/cmd/go/internal/test/flagdefs.go b/libgo/go/cmd/go/internal/test/flagdefs.go index 37ac81c2678..1b79314eff3 100644 --- a/libgo/go/cmd/go/internal/test/flagdefs.go +++ b/libgo/go/cmd/go/internal/test/flagdefs.go @@ -19,6 +19,9 @@ var passFlagToTest = map[string]bool{ "cpu": true, "cpuprofile": true, "failfast": true, + "fuzz": true, + "fuzzminimizetime": true, + "fuzztime": true, "list": true, "memprofile": true, "memprofilerate": true, @@ -33,3 +36,37 @@ var passFlagToTest = map[string]bool{ "trace": true, "v": true, } + +var passAnalyzersToVet = map[string]bool{ + "asmdecl": true, + "assign": true, + "atomic": true, + "bool": true, + "bools": true, + "buildtag": true, + "buildtags": true, + "cgocall": true, + "composites": true, + "copylocks": true, + "errorsas": true, + "framepointer": true, + "httpresponse": true, + "ifaceassert": true, + "loopclosure": true, + "lostcancel": true, + "methods": true, + "nilfunc": true, + "printf": true, + "rangeloops": true, + "shift": true, + "sigchanyzer": true, + "stdmethods": true, + "stringintconv": true, + "structtag": true, + "testinggoroutine": true, + "tests": true, + "unmarshal": true, + "unreachable": true, + "unsafeptr": true, + "unusedresult": true, +} diff --git a/libgo/go/cmd/go/internal/test/flagdefs_test.go b/libgo/go/cmd/go/internal/test/flagdefs_test.go index ab5440b3801..a2f09e5a960 100644 --- a/libgo/go/cmd/go/internal/test/flagdefs_test.go +++ b/libgo/go/cmd/go/internal/test/flagdefs_test.go @@ -5,7 +5,10 @@ package test import ( + "cmd/go/internal/test/internal/genflags" "flag" + "internal/testenv" + "reflect" "strings" "testing" ) @@ -17,7 +20,7 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { } name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These are internal flags. default: if !passFlagToTest[name] { @@ -37,3 +40,22 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { } } } + +func TestVetAnalyzersSetIsCorrect(t *testing.T) { + testenv.MustHaveGoBuild(t) + + vetAns, err := genflags.VetAnalyzers() + if err != nil { + t.Fatal(err) + } + + want := make(map[string]bool) + for _, a := range vetAns { + want[a] = true + } + + if !reflect.DeepEqual(want, passAnalyzersToVet) { + t.Errorf("stale vet analyzers: want %v; got %v", want, passAnalyzersToVet) + t.Logf("(Run 'go generate cmd/go/internal/test' to refresh the set of analyzers.)") + } +} diff --git a/libgo/go/cmd/go/internal/test/genflags.go b/libgo/go/cmd/go/internal/test/genflags.go index 9277de7fee8..10f290090c7 100644 --- a/libgo/go/cmd/go/internal/test/genflags.go +++ b/libgo/go/cmd/go/internal/test/genflags.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ignore -// +build ignore package main @@ -16,6 +15,8 @@ import ( "strings" "testing" "text/template" + + "cmd/go/internal/test/internal/genflags" ) func main() { @@ -25,9 +26,18 @@ func main() { } func regenerate() error { + vetAnalyzers, err := genflags.VetAnalyzers() + if err != nil { + return err + } + t := template.Must(template.New("fileTemplate").Parse(fileTemplate)) + tData := map[string][]string{ + "testFlags": testFlags(), + "vetAnalyzers": vetAnalyzers, + } buf := bytes.NewBuffer(nil) - if err := t.Execute(buf, testFlags()); err != nil { + if err := t.Execute(buf, tData); err != nil { return err } @@ -64,7 +74,7 @@ func testFlags() []string { name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These flags are only for use by cmd/go. default: names = append(names, name) @@ -85,7 +95,13 @@ package test // passFlagToTest contains the flags that should be forwarded to // the test binary with the prefix "test.". var passFlagToTest = map[string]bool { -{{- range .}} +{{- range .testFlags}} + "{{.}}": true, +{{- end }} +} + +var passAnalyzersToVet = map[string]bool { +{{- range .vetAnalyzers}} "{{.}}": true, {{- end }} } diff --git a/libgo/go/cmd/go/internal/test/internal/genflags/vetflag.go b/libgo/go/cmd/go/internal/test/internal/genflags/vetflag.go new file mode 100644 index 00000000000..2195cc34479 --- /dev/null +++ b/libgo/go/cmd/go/internal/test/internal/genflags/vetflag.go @@ -0,0 +1,68 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package genflags + +import ( + "bytes" + "cmd/go/internal/base" + "encoding/json" + "fmt" + exec "internal/execabs" + "regexp" + "sort" +) + +// VetAnalyzers computes analyzers and their aliases supported by vet. +func VetAnalyzers() ([]string, error) { + // get supported vet flag information + tool := base.Tool("vet") + vetcmd := exec.Command(tool, "-flags") + out := new(bytes.Buffer) + vetcmd.Stdout = out + if err := vetcmd.Run(); err != nil { + return nil, fmt.Errorf("go vet: can't execute %s -flags: %v\n", tool, err) + } + var analysisFlags []struct { + Name string + Bool bool + Usage string + } + if err := json.Unmarshal(out.Bytes(), &analysisFlags); err != nil { + return nil, fmt.Errorf("go vet: can't unmarshal JSON from %s -flags: %v", tool, err) + } + + // parse the flags to figure out which ones stand for analyses + analyzerSet := make(map[string]bool) + rEnable := regexp.MustCompile("^enable .+ analysis$") + for _, flag := range analysisFlags { + if rEnable.MatchString(flag.Usage) { + analyzerSet[flag.Name] = true + } + } + + rDeprecated := regexp.MustCompile("^deprecated alias for -(?P<analyzer>(.+))$") + // Returns the original value matched by rDeprecated on input value. + // If there is no match, "" is returned. + originalValue := func(value string) string { + match := rDeprecated.FindStringSubmatch(value) + if len(match) < 2 { + return "" + } + return match[1] + } + // extract deprecated aliases for existing analyses + for _, flag := range analysisFlags { + if o := originalValue(flag.Usage); analyzerSet[o] { + analyzerSet[flag.Name] = true + } + } + + var analyzers []string + for a := range analyzerSet { + analyzers = append(analyzers, a) + } + sort.Strings(analyzers) + return analyzers, nil +} diff --git a/libgo/go/cmd/go/internal/test/test.go b/libgo/go/cmd/go/internal/test/test.go index 59ea1ef5445..50e6d5201b0 100644 --- a/libgo/go/cmd/go/internal/test/test.go +++ b/libgo/go/cmd/go/internal/test/test.go @@ -29,11 +29,15 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" + "cmd/go/internal/modload" "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" "cmd/go/internal/work" + "cmd/internal/sys" "cmd/internal/test2json" + + "golang.org/x/mod/module" ) // Break init loop. @@ -60,8 +64,8 @@ followed by detailed output for each failed package. 'Go test' recompiles each package along with any files with names matching the file pattern "*_test.go". -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. +These additional files can contain test functions, benchmark functions, fuzz +tests and example functions. See 'go help testfunc' for more. Each listed package causes the execution of a separate test binary. Files whose names begin with "_" (including "_test.go") or "." are ignored. @@ -78,7 +82,8 @@ binary. Only a high-confidence subset of the default go vet checks are used. That subset is: 'atomic', 'bool', 'buildtags', 'errorsas', 'ifaceassert', 'nilfunc', 'printf', and 'stringintconv'. You can see the documentation for these and other vet tests via "go doc cmd/vet". -To disable the running of go vet, use the -vet=off flag. +To disable the running of go vet, use the -vet=off flag. To run all +checks, use the -vet=all flag. All test output and summary lines are printed to the go command's standard output, even if the test printed them to its own standard @@ -119,16 +124,16 @@ elapsed time in the summary line. The rule for a match in the cache is that the run involves the same test binary and the flags on the command line come entirely from a restricted set of 'cacheable' test flags, defined as -benchtime, -cpu, --list, -parallel, -run, -short, and -v. If a run of go test has any test -or non-test flags outside this set, the result is not cached. To -disable test caching, use any test flag or argument other than the -cacheable flags. The idiomatic way to disable test caching explicitly -is to use -count=1. Tests that open files within the package's source -root (usually $GOPATH) or that consult environment variables only -match future runs in which the files and environment variables are unchanged. -A cached test result is treated as executing in no time at all, -so a successful package test result will be cached and reused -regardless of -timeout setting. +-list, -parallel, -run, -short, -timeout, -failfast, and -v. +If a run of go test has any test or non-test flags outside this set, +the result is not cached. To disable test caching, use any test flag +or argument other than the cacheable flags. The idiomatic way to disable +test caching explicitly is to use -count=1. Tests that open files within +the package's source root (usually $GOPATH) or that consult environment +variables only match future runs in which the files and environment +variables are unchanged. A cached test result is treated as executing +in no time at all,so a successful package test result will be cached and +reused regardless of -timeout setting. In addition to the build flags, the flags handled by 'go test' itself are: @@ -206,9 +211,10 @@ control the execution of any test: (for example, -benchtime 100x). -count n - Run each test and benchmark n times (default 1). + Run each test, benchmark, and fuzz seed n times (default 1). If -cpu is set, run n times for each GOMAXPROCS value. - Examples are always run once. + Examples are always run once. -count does not apply to + fuzz tests matched by -fuzz. -cover Enable coverage analysis. @@ -235,32 +241,67 @@ control the execution of any test: Sets -cover. -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value - of GOMAXPROCS. + Specify a list of GOMAXPROCS values for which the tests, benchmarks or + fuzz tests should be executed. The default is the current value + of GOMAXPROCS. -cpu does not apply to fuzz tests matched by -fuzz. -failfast Do not start new tests after the first test failure. + -fuzz regexp + Run the fuzz test matching the regular expression. When specified, + the command line argument must match exactly one package within the + main module, and regexp must match exactly one fuzz test within + that package. Fuzzing will occur after tests, benchmarks, seed corpora + of other fuzz tests, and examples have completed. See the Fuzzing + section of the testing package documentation for details. + + -fuzztime t + Run enough iterations of the fuzz target during fuzzing to take t, + specified as a time.Duration (for example, -fuzztime 1h30s). + The default is to run forever. + The special syntax Nx means to run the fuzz target N times + (for example, -fuzztime 1000x). + + -fuzzminimizetime t + Run enough iterations of the fuzz target during each minimization + attempt to take t, as specified as a time.Duration (for example, + -fuzzminimizetime 30s). + The default is 60s. + The special syntax Nx means to run the fuzz target N times + (for example, -fuzzminimizetime 100x). + + -json + Log verbose output and test results in JSON. This presents the + same information as the -v flag in a machine-readable format. + -list regexp - List tests, benchmarks, or examples matching the regular expression. - No tests, benchmarks or examples will be run. This will only - list top-level tests. No subtest or subbenchmarks will be shown. + List tests, benchmarks, fuzz tests, or examples matching the regular + expression. No tests, benchmarks, fuzz tests, or examples will be run. + This will only list top-level tests. No subtest or subbenchmarks will be + shown. -parallel n - Allow parallel execution of test functions that call t.Parallel. + Allow parallel execution of test functions that call t.Parallel, and + fuzz targets that call t.Parallel when running the seed corpus. The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. + simultaneously. + While fuzzing, the value of this flag is the maximum number of + subprocesses that may call the fuzz function simultaneously, regardless of + whether T.Parallel is called. + By default, -parallel is set to the value of GOMAXPROCS. + Setting -parallel to values higher than GOMAXPROCS may cause degraded + performance due to CPU contention, especially when fuzzing. Note that -parallel only applies within a single test binary. The 'go test' command may run tests for different packages in parallel as well, according to the setting of the -p flag (see 'go help build'). -run regexp - Run only those tests and examples matching the regular expression. - For tests, the regular expression is split by unbracketed slash (/) - characters into a sequence of regular expressions, and each part - of a test's identifier must match the corresponding element in + Run only those tests, examples, and fuzz tests matching the regular + expression. For tests, the regular expression is split by unbracketed + slash (/) characters into a sequence of regular expressions, and each + part of a test's identifier must match the corresponding element in the sequence, if any. Note that possible parents of matches are run too, so that -run=X/Y matches and runs and reports the result of all tests matching X, even those without sub-tests matching Y, @@ -273,11 +314,11 @@ control the execution of any test: exhaustive tests. -shuffle off,on,N - Randomize the execution order of tests and benchmarks. - It is off by default. If -shuffle is set to on, then it will seed - the randomizer using the system clock. If -shuffle is set to an - integer N, then N will be used as the seed value. In both cases, - the seed will be reported for reproducibility. + Randomize the execution order of tests and benchmarks. + It is off by default. If -shuffle is set to on, then it will seed + the randomizer using the system clock. If -shuffle is set to an + integer N, then N will be used as the seed value. In both cases, + the seed will be reported for reproducibility. -timeout d If a test binary runs longer than duration d, panic. @@ -373,7 +414,11 @@ leave the test binary in pkg.test for use when analyzing the profiles. When 'go test' runs a test binary, it does so from within the corresponding package's source code directory. Depending on the test, it may be necessary to do the same when invoking a generated test -binary directly. +binary directly. Because that directory may be located within the +module cache, which may be read-only and is verified by checksums, the +test must not write to it or any other directory within the module +unless explicitly requested by the user (such as with the -fuzz flag, +which writes failures to testdata/fuzz). The command-line package list, if present, must appear before any flag not known to the go test command. Continuing the example above, @@ -430,6 +475,10 @@ A benchmark function is one named BenchmarkXxx and should have the signature, func BenchmarkXxx(b *testing.B) { ... } +A fuzz test is one named FuzzXxx and should have the signature, + + func FuzzXxx(f *testing.F) { ... } + An example function is similar to a test function but, instead of using *testing.T to report success or failure, prints output to os.Stdout. If the last comment in the function starts with "Output:" then the output @@ -469,7 +518,7 @@ Here is another example where the ordering of the output is ignored: The entire test file is presented as the example when it contains a single example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. +declaration, and no tests, benchmarks, or fuzz tests. See the documentation of the testing package for more information. `, @@ -483,6 +532,7 @@ var ( testCoverPaths []string // -coverpkg flag testCoverPkgs []*load.Package // -coverpkg flag testCoverProfile string // -coverprofile flag + testFuzz string // -fuzz flag testJSON bool // -json flag testList string // -list flag testO string // -o flag @@ -578,6 +628,7 @@ var defaultVetFlags = []string{ func runTest(ctx context.Context, cmd *base.Command, args []string) { pkgArgs, testArgs = testFlags(args) + modload.InitWorkfile() // The test command does custom flag processing; initialize workspaces after that. if cfg.DebugTrace != "" { var close func() error @@ -615,6 +666,52 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { if testO != "" && len(pkgs) != 1 { base.Fatalf("cannot use -o flag with multiple packages") } + if testFuzz != "" { + if !sys.FuzzSupported(cfg.Goos, cfg.Goarch) { + base.Fatalf("-fuzz flag is not supported on %s/%s", cfg.Goos, cfg.Goarch) + } + if len(pkgs) != 1 { + base.Fatalf("cannot use -fuzz flag with multiple packages") + } + if testCoverProfile != "" { + base.Fatalf("cannot use -coverprofile flag with -fuzz flag") + } + if profileFlag := testProfile(); profileFlag != "" { + base.Fatalf("cannot use %s flag with -fuzz flag", profileFlag) + } + + // Reject the '-fuzz' flag if the package is outside the main module. + // Otherwise, if fuzzing identifies a failure it could corrupt checksums in + // the module cache (or permanently alter the behavior of std tests for all + // users) by writing the failing input to the package's testdata directory. + // (See https://golang.org/issue/48495 and test_fuzz_modcache.txt.) + mainMods := modload.MainModules + if m := pkgs[0].Module; m != nil && m.Path != "" { + if !mainMods.Contains(m.Path) { + base.Fatalf("cannot use -fuzz flag on package outside the main module") + } + } else if pkgs[0].Standard && modload.Enabled() { + // Because packages in 'std' and 'cmd' are part of the standard library, + // they are only treated as part of a module in 'go mod' subcommands and + // 'go get'. However, we still don't want to accidentally corrupt their + // testdata during fuzzing, nor do we want to fail with surprising errors + // if GOROOT isn't writable (as is often the case for Go toolchains + // installed through package managers). + // + // If the user is requesting to fuzz a standard-library package, ensure + // that they are in the same module as that package (just like when + // fuzzing any other package). + if strings.HasPrefix(pkgs[0].ImportPath, "cmd/") { + if !mainMods.Contains("cmd") || !mainMods.InGorootSrc(module.Version{Path: "cmd"}) { + base.Fatalf("cannot use -fuzz flag on package outside the main module") + } + } else { + if !mainMods.Contains("std") || !mainMods.InGorootSrc(module.Version{Path: "std"}) { + base.Fatalf("cannot use -fuzz flag on package outside the main module") + } + } + } + } if testProfile() != "" && len(pkgs) != 1 { base.Fatalf("cannot use %s flag with multiple packages", testProfile()) } @@ -625,7 +722,9 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { // to that timeout plus one minute. This is a backup alarm in case // the test wedges with a goroutine spinning and its background // timer does not get a chance to fire. - if testTimeout > 0 { + // Don't set this if fuzzing, since it should be able to run + // indefinitely. + if testTimeout > 0 && testFuzz == "" { testKillTimeout = testTimeout + 1*time.Minute } @@ -649,7 +748,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { b.Init() if cfg.BuildI { - fmt.Fprint(os.Stderr, "go test: -i flag is deprecated\n") + fmt.Fprint(os.Stderr, "go: -i flag is deprecated\n") cfg.BuildV = testV deps := make(map[string]bool) @@ -775,6 +874,41 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } } + // Inform the compiler that it should instrument the binary at + // build-time when fuzzing is enabled. + if testFuzz != "" { + // Don't instrument packages which may affect coverage guidance but are + // unlikely to be useful. Most of these are used by the testing or + // internal/fuzz packages concurrently with fuzzing. + var skipInstrumentation = map[string]bool{ + "context": true, + "internal/fuzz": true, + "reflect": true, + "runtime": true, + "sync": true, + "sync/atomic": true, + "syscall": true, + "testing": true, + "time": true, + } + for _, p := range load.TestPackageList(ctx, pkgOpts, pkgs) { + if !skipInstrumentation[p.ImportPath] { + p.Internal.FuzzInstrument = true + } + } + } + + // Collect all the packages imported by the packages being tested. + allImports := make(map[*load.Package]bool) + for _, p := range pkgs { + if p.Error != nil && p.Error.IsImportCycle { + continue + } + for _, p1 := range p.Internal.Imports { + allImports[p1] = true + } + } + // Prepare build + run + print actions for all packages being tested. for _, p := range pkgs { // sync/atomic import is inserted by the cover tool. See #18486 @@ -782,7 +916,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { ensureImport(p, "sync/atomic") } - buildTest, runTest, printTest, err := builderTest(&b, ctx, pkgOpts, p) + buildTest, runTest, printTest, err := builderTest(&b, ctx, pkgOpts, p, allImports[p]) if err != nil { str := err.Error() str = strings.TrimPrefix(str, "\n") @@ -849,7 +983,7 @@ var windowsBadWords = []string{ "update", } -func builderTest(b *work.Builder, ctx context.Context, pkgOpts load.PackageOpts, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { +func builderTest(b *work.Builder, ctx context.Context, pkgOpts load.PackageOpts, p *load.Package, imported bool) (buildAction, runAction, printAction *work.Action, err error) { if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { build := b.CompileAction(work.ModeBuild, work.ModeBuild, p) run := &work.Action{Mode: "test run", Package: p, Deps: []*work.Action{build}} @@ -877,6 +1011,16 @@ func builderTest(b *work.Builder, ctx context.Context, pkgOpts load.PackageOpts, return nil, nil, nil, err } + // If imported is true then this package is imported by some + // package being tested. Make building the test version of the + // package depend on building the non-test version, so that we + // only report build errors once. Issue #44624. + if imported && ptest != p { + buildTest := b.CompileAction(work.ModeBuild, work.ModeBuild, ptest) + buildP := b.CompileAction(work.ModeBuild, work.ModeBuild, p) + buildTest.Deps = append(buildTest.Deps, buildP) + } + // Use last element of import path, not package name. // They differ when package name is "main". // But if the import path is "command-line-arguments", @@ -1080,6 +1224,8 @@ func declareCoverVars(p *load.Package, files ...string) map[string]*load.CoverVa } var noTestsToRun = []byte("\ntesting: warning: no tests to run\n") +var noFuzzTestsToFuzz = []byte("\ntesting: warning: no fuzz tests to fuzz\n") +var tooManyFuzzTestsToFuzz = []byte("\ntesting: warning: -fuzz matches more than one fuzz test, won't fuzz\n") type runCache struct { disableCache bool // cache should be disabled for this run @@ -1127,10 +1273,10 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } var buf bytes.Buffer - if len(pkgArgs) == 0 || (testBench != "") { + if len(pkgArgs) == 0 || testBench != "" || testFuzz != "" { // Stream test output (no buffering) when no package has // been given on the command line (implicit current directory) - // or when benchmarking. + // or when benchmarking or fuzzing. // No change to stdout. } else { // If we're only running a single package under test or if parallelism is @@ -1183,7 +1329,12 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } panicArg := "-test.paniconexit0" - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) + fuzzArg := []string{} + if testFuzz != "" { + fuzzCacheDir := filepath.Join(cache.Default().FuzzDir(), a.Package.ImportPath) + fuzzArg = []string{"-test.fuzzcachedir=" + fuzzCacheDir} + } + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, fuzzArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. @@ -1276,15 +1427,31 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { norun = " [no tests to run]" } + if bytes.HasPrefix(out, noFuzzTestsToFuzz[1:]) || bytes.Contains(out, noFuzzTestsToFuzz) { + norun = " [no fuzz tests to fuzz]" + } + if bytes.HasPrefix(out, tooManyFuzzTestsToFuzz[1:]) || bytes.Contains(out, tooManyFuzzTestsToFuzz) { + norun = "[-fuzz matches more than one fuzz test, won't fuzz]" + } + if len(out) > 0 && !bytes.HasSuffix(out, []byte("\n")) { + // Ensure that the output ends with a newline before the "ok" + // line we're about to print (https://golang.org/issue/49317). + cmd.Stdout.Write([]byte("\n")) + } fmt.Fprintf(cmd.Stdout, "ok \t%s\t%s%s%s\n", a.Package.ImportPath, t, coveragePercentage(out), norun) c.saveOutput(a) } else { base.SetExitStatus(1) - // If there was test output, assume we don't need to print the exit status. - // Buf there's no test output, do print the exit status. if len(out) == 0 { + // If there was no test output, print the exit status so that the reason + // for failure is clear. fmt.Fprintf(cmd.Stdout, "%s\n", err) + } else if !bytes.HasSuffix(out, []byte("\n")) { + // Otherwise, ensure that the output ends with a newline before the FAIL + // line we're about to print (https://golang.org/issue/49317). + cmd.Stdout.Write([]byte("\n")) } + // NOTE(golang.org/issue/37555): test2json reports that a test passes // unless "FAIL" is printed at the beginning of a line. The test may not // actually print that if it panics, exits, or terminates abnormally, @@ -1347,6 +1514,7 @@ func (c *runCache) tryCacheWithID(b *work.Builder, a *work.Action, id string) bo "-test.run", "-test.short", "-test.timeout", + "-test.failfast", "-test.v": // These are cacheable. // Note that this list is documented above, @@ -1711,9 +1879,23 @@ func builderNoTest(b *work.Builder, ctx context.Context, a *work.Action) error { return nil } -// printExitStatus is the action for printing the exit status +// printExitStatus is the action for printing the final exit status. +// If we are running multiple test targets, print a final "FAIL" +// in case a failure in an early package has already scrolled +// off of the user's terminal. +// (See https://golang.org/issue/30507#issuecomment-470593235.) +// +// In JSON mode, we need to maintain valid JSON output and +// we assume that the test output is being parsed by a tool +// anyway, so the failure will not be missed and would be +// awkward to try to wedge into the JSON stream. +// +// In fuzz mode, we only allow a single package for now +// (see CL 350156 and https://golang.org/issue/46312), +// so there is no possibility of scrolling off and no need +// to print the final status. func printExitStatus(b *work.Builder, ctx context.Context, a *work.Action) error { - if !testJSON && len(pkgArgs) != 0 { + if !testJSON && testFuzz == "" && len(pkgArgs) != 0 { if base.GetExitStatus() != 0 { fmt.Println("FAIL") return nil diff --git a/libgo/go/cmd/go/internal/test/testflag.go b/libgo/go/cmd/go/internal/test/testflag.go index 08f1efa2c0d..b9d1ec91ff7 100644 --- a/libgo/go/cmd/go/internal/test/testflag.go +++ b/libgo/go/cmd/go/internal/test/testflag.go @@ -5,6 +5,10 @@ package test import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/cmdflag" + "cmd/go/internal/work" "errors" "flag" "fmt" @@ -13,11 +17,6 @@ import ( "strconv" "strings" "time" - - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/cmdflag" - "cmd/go/internal/work" ) //go:generate go run ./genflags.go @@ -29,6 +28,7 @@ import ( func init() { work.AddBuildFlags(CmdTest, work.OmitVFlag) + base.AddWorkfileFlag(&CmdTest.Flag) cf := CmdTest.Flag cf.BoolVar(&testC, "c", false, "") @@ -57,6 +57,7 @@ func init() { cf.String("cpu", "", "") cf.StringVar(&testCPUProfile, "cpuprofile", "", "") cf.Bool("failfast", false, "") + cf.StringVar(&testFuzz, "fuzz", "", "") cf.StringVar(&testList, "list", "", "") cf.StringVar(&testMemProfile, "memprofile", "", "") cf.String("memprofilerate", "", "") @@ -67,6 +68,8 @@ func init() { cf.String("run", "", "") cf.Bool("short", false, "") cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "") + cf.String("fuzztime", "", "") + cf.String("fuzzminimizetime", "", "") cf.StringVar(&testTrace, "trace", "", "") cf.BoolVar(&testV, "v", false, "") cf.Var(&testShuffle, "shuffle", "") @@ -134,6 +137,7 @@ type outputdirFlag struct { func (f *outputdirFlag) String() string { return f.abs } + func (f *outputdirFlag) Set(value string) (err error) { if value == "" { f.abs = "" @@ -142,6 +146,7 @@ func (f *outputdirFlag) Set(value string) (err error) { } return err } + func (f *outputdirFlag) getAbs() string { if f.abs == "" { return base.Cwd() @@ -150,8 +155,12 @@ func (f *outputdirFlag) getAbs() string { } // vetFlag implements the special parsing logic for the -vet flag: -// a comma-separated list, with a distinguished value "off" and -// a boolean tracking whether it was set explicitly. +// a comma-separated list, with distinguished values "all" and +// "off", plus a boolean tracking whether it was set explicitly. +// +// "all" is encoded as vetFlag{true, false, nil}, since it will +// pass no flags to the vet binary, and by default, it runs all +// analyzers. type vetFlag struct { explicit bool off bool @@ -159,7 +168,10 @@ type vetFlag struct { } func (f *vetFlag) String() string { - if f.off { + switch { + case !f.off && !f.explicit && len(f.flags) == 0: + return "all" + case f.off: return "off" } @@ -174,31 +186,45 @@ func (f *vetFlag) String() string { } func (f *vetFlag) Set(value string) error { - if value == "" { + switch { + case value == "": *f = vetFlag{flags: defaultVetFlags} return nil - } - - if value == "off" { - *f = vetFlag{ - explicit: true, - off: true, - } - return nil - } - - if strings.Contains(value, "=") { + case strings.Contains(value, "="): return fmt.Errorf("-vet argument cannot contain equal signs") - } - if strings.Contains(value, " ") { + case strings.Contains(value, " "): return fmt.Errorf("-vet argument is comma-separated list, cannot contain spaces") } + *f = vetFlag{explicit: true} + var single string for _, arg := range strings.Split(value, ",") { - if arg == "" { + switch arg { + case "": return fmt.Errorf("-vet argument contains empty list element") + case "all": + single = arg + *f = vetFlag{explicit: true} + continue + case "off": + single = arg + *f = vetFlag{ + explicit: true, + off: true, + } + continue + default: + if _, ok := passAnalyzersToVet[arg]; !ok { + return fmt.Errorf("-vet argument must be a supported analyzer or a distinguished value; found %s", arg) + } + f.flags = append(f.flags, "-"+arg) } - f.flags = append(f.flags, "-"+arg) + } + if len(f.flags) > 1 && single != "" { + return fmt.Errorf("-vet does not accept %q in a list with other analyzers", single) + } + if len(f.flags) > 1 && single != "" { + return fmt.Errorf("-vet does not accept %q in a list with other analyzers", single) } return nil } @@ -369,7 +395,7 @@ func testFlags(args []string) (packageNames, passToTest []string) { if !testC { buildFlag = "-i" } - fmt.Fprintf(os.Stderr, "go test: unknown flag %s cannot be used with %s\n", firstUnknownFlag, buildFlag) + fmt.Fprintf(os.Stderr, "go: unknown flag %s cannot be used with %s\n", firstUnknownFlag, buildFlag) exitWithUsage() } diff --git a/libgo/go/cmd/go/internal/tool/tool.go b/libgo/go/cmd/go/internal/tool/tool.go index 95c90ea7c8d..4fe4c2baeda 100644 --- a/libgo/go/cmd/go/internal/tool/tool.go +++ b/libgo/go/cmd/go/internal/tool/tool.go @@ -61,7 +61,7 @@ func runTool(ctx context.Context, cmd *base.Command, args []string) { switch { case 'a' <= c && c <= 'z', '0' <= c && c <= '9', c == '_': default: - fmt.Fprintf(os.Stderr, "go tool: bad tool name %q\n", toolName) + fmt.Fprintf(os.Stderr, "go: bad tool name %q\n", toolName) base.SetExitStatus(2) return } @@ -117,14 +117,14 @@ func runTool(ctx context.Context, cmd *base.Command, args []string) { func listTools() { f, err := os.Open(base.ToolDir) if err != nil { - fmt.Fprintf(os.Stderr, "go tool: no tool directory: %s\n", err) + fmt.Fprintf(os.Stderr, "go: no tool directory: %s\n", err) base.SetExitStatus(2) return } defer f.Close() names, err := f.Readdirnames(-1) if err != nil { - fmt.Fprintf(os.Stderr, "go tool: can't read directory: %s\n", err) + fmt.Fprintf(os.Stderr, "go: can't read tool directory: %s\n", err) base.SetExitStatus(2) return } diff --git a/libgo/go/cmd/go/internal/txtar/archive.go b/libgo/go/cmd/go/internal/txtar/archive.go deleted file mode 100644 index 17966848771..00000000000 --- a/libgo/go/cmd/go/internal/txtar/archive.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package txtar implements a trivial text-based file archive format. -// -// The goals for the format are: -// -// - be trivial enough to create and edit by hand. -// - be able to store trees of text files describing go command test cases. -// - diff nicely in git history and code reviews. -// -// Non-goals include being a completely general archive format, -// storing binary data, storing file modes, storing special files like -// symbolic links, and so on. -// -// Txtar format -// -// A txtar archive is zero or more comment lines and then a sequence of file entries. -// Each file entry begins with a file marker line of the form "-- FILENAME --" -// and is followed by zero or more file content lines making up the file data. -// The comment or file content ends at the next file marker line. -// The file marker line must begin with the three-byte sequence "-- " -// and end with the three-byte sequence " --", but the enclosed -// file name can be surrounding by additional white space, -// all of which is stripped. -// -// If the txtar file is missing a trailing newline on the final line, -// parsers should consider a final newline to be present anyway. -// -// There are no possible syntax errors in a txtar archive. -package txtar - -import ( - "bytes" - "fmt" - "os" - "strings" -) - -// An Archive is a collection of files. -type Archive struct { - Comment []byte - Files []File -} - -// A File is a single file in an archive. -type File struct { - Name string // name of file ("foo/bar.txt") - Data []byte // text content of file -} - -// Format returns the serialized form of an Archive. -// It is assumed that the Archive data structure is well-formed: -// a.Comment and all a.File[i].Data contain no file marker lines, -// and all a.File[i].Name is non-empty. -func Format(a *Archive) []byte { - var buf bytes.Buffer - buf.Write(fixNL(a.Comment)) - for _, f := range a.Files { - fmt.Fprintf(&buf, "-- %s --\n", f.Name) - buf.Write(fixNL(f.Data)) - } - return buf.Bytes() -} - -// ParseFile parses the named file as an archive. -func ParseFile(file string) (*Archive, error) { - data, err := os.ReadFile(file) - if err != nil { - return nil, err - } - return Parse(data), nil -} - -// Parse parses the serialized form of an Archive. -// The returned Archive holds slices of data. -func Parse(data []byte) *Archive { - a := new(Archive) - var name string - a.Comment, name, data = findFileMarker(data) - for name != "" { - f := File{name, nil} - f.Data, name, data = findFileMarker(data) - a.Files = append(a.Files, f) - } - return a -} - -var ( - newlineMarker = []byte("\n-- ") - marker = []byte("-- ") - markerEnd = []byte(" --") -) - -// findFileMarker finds the next file marker in data, -// extracts the file name, and returns the data before the marker, -// the file name, and the data after the marker. -// If there is no next marker, findFileMarker returns before = fixNL(data), name = "", after = nil. -func findFileMarker(data []byte) (before []byte, name string, after []byte) { - var i int - for { - if name, after = isMarker(data[i:]); name != "" { - return data[:i], name, after - } - j := bytes.Index(data[i:], newlineMarker) - if j < 0 { - return fixNL(data), "", nil - } - i += j + 1 // positioned at start of new possible marker - } -} - -// isMarker checks whether data begins with a file marker line. -// If so, it returns the name from the line and the data after the line. -// Otherwise it returns name == "" with an unspecified after. -func isMarker(data []byte) (name string, after []byte) { - if !bytes.HasPrefix(data, marker) { - return "", nil - } - if i := bytes.IndexByte(data, '\n'); i >= 0 { - data, after = data[:i], data[i+1:] - } - if !bytes.HasSuffix(data, markerEnd) { - return "", nil - } - return strings.TrimSpace(string(data[len(marker) : len(data)-len(markerEnd)])), after -} - -// If data is empty or ends in \n, fixNL returns data. -// Otherwise fixNL returns a new slice consisting of data with a final \n added. -func fixNL(data []byte) []byte { - if len(data) == 0 || data[len(data)-1] == '\n' { - return data - } - d := make([]byte, len(data)+1) - copy(d, data) - d[len(data)] = '\n' - return d -} diff --git a/libgo/go/cmd/go/internal/txtar/archive_test.go b/libgo/go/cmd/go/internal/txtar/archive_test.go deleted file mode 100644 index 3f734f67625..00000000000 --- a/libgo/go/cmd/go/internal/txtar/archive_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package txtar - -import ( - "bytes" - "fmt" - "reflect" - "testing" -) - -var tests = []struct { - name string - text string - parsed *Archive -}{ - { - name: "basic", - text: `comment1 -comment2 --- file1 -- -File 1 text. --- foo --- -More file 1 text. --- file 2 -- -File 2 text. --- empty -- --- noNL -- -hello world`, - parsed: &Archive{ - Comment: []byte("comment1\ncomment2\n"), - Files: []File{ - {"file1", []byte("File 1 text.\n-- foo ---\nMore file 1 text.\n")}, - {"file 2", []byte("File 2 text.\n")}, - {"empty", []byte{}}, - {"noNL", []byte("hello world\n")}, - }, - }, - }, -} - -func Test(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := Parse([]byte(tt.text)) - if !reflect.DeepEqual(a, tt.parsed) { - t.Fatalf("Parse: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) - } - text := Format(a) - a = Parse(text) - if !reflect.DeepEqual(a, tt.parsed) { - t.Fatalf("Parse after Format: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) - } - }) - } -} - -func shortArchive(a *Archive) string { - var buf bytes.Buffer - fmt.Fprintf(&buf, "comment: %q\n", a.Comment) - for _, f := range a.Files { - fmt.Fprintf(&buf, "file %q: %q\n", f.Name, f.Data) - } - return buf.String() -} diff --git a/libgo/go/cmd/go/internal/vcs/vcs.go b/libgo/go/cmd/go/internal/vcs/vcs.go index 91485f6f745..fd521b2eb15 100644 --- a/libgo/go/cmd/go/internal/vcs/vcs.go +++ b/libgo/go/cmd/go/internal/vcs/vcs.go @@ -5,7 +5,7 @@ package vcs import ( - "encoding/json" + "bytes" "errors" "fmt" exec "internal/execabs" @@ -17,8 +17,10 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "sync" + "time" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -29,11 +31,12 @@ import ( "golang.org/x/mod/module" ) -// A vcsCmd describes how to use a version control system +// A Cmd describes how to use a version control system // like Mercurial, Git, or Subversion. type Cmd struct { - Name string - Cmd string // name of binary to invoke command + Name string + Cmd string // name of binary to invoke command + RootNames []string // filename indicating the root of a checkout directory CreateCmd []string // commands to download a fresh copy of a repository DownloadCmd []string // commands to download updates into an existing repository @@ -48,6 +51,14 @@ type Cmd struct { RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) + Status func(v *Cmd, rootDir string) (Status, error) +} + +// Status is the current state of a local repository. +type Status struct { + Revision string // Optional. + CommitTime time.Time // Optional. + Uncommitted bool // Required. } var defaultSecureScheme = map[string]bool{ @@ -118,8 +129,9 @@ func vcsByCmd(cmd string) *Cmd { // vcsHg describes how to use Mercurial. var vcsHg = &Cmd{ - Name: "Mercurial", - Cmd: "hg", + Name: "Mercurial", + Cmd: "hg", + RootNames: []string{".hg"}, CreateCmd: []string{"clone -U -- {repo} {dir}"}, DownloadCmd: []string{"pull"}, @@ -139,6 +151,7 @@ var vcsHg = &Cmd{ Scheme: []string{"https", "http", "ssh"}, PingCmd: "identify -- {scheme}://{repo}", RemoteRepo: hgRemoteRepo, + Status: hgStatus, } func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { @@ -149,10 +162,64 @@ func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { return strings.TrimSpace(string(out)), nil } +func hgStatus(vcsHg *Cmd, rootDir string) (Status, error) { + // Output changeset ID and seconds since epoch. + out, err := vcsHg.runOutputVerboseOnly(rootDir, `log -l1 -T {node}:{date|hgdate}`) + if err != nil { + return Status{}, err + } + + // Successful execution without output indicates an empty repo (no commits). + var rev string + var commitTime time.Time + if len(out) > 0 { + // Strip trailing timezone offset. + if i := bytes.IndexByte(out, ' '); i > 0 { + out = out[:i] + } + rev, commitTime, err = parseRevTime(out) + if err != nil { + return Status{}, err + } + } + + // Also look for untracked files. + out, err = vcsHg.runOutputVerboseOnly(rootDir, "status") + if err != nil { + return Status{}, err + } + uncommitted := len(out) > 0 + + return Status{ + Revision: rev, + CommitTime: commitTime, + Uncommitted: uncommitted, + }, nil +} + +// parseRevTime parses commit details in "revision:seconds" format. +func parseRevTime(out []byte) (string, time.Time, error) { + buf := string(bytes.TrimSpace(out)) + + i := strings.IndexByte(buf, ':') + if i < 1 { + return "", time.Time{}, errors.New("unrecognized VCS tool output") + } + rev := buf[:i] + + secs, err := strconv.ParseInt(string(buf[i+1:]), 10, 64) + if err != nil { + return "", time.Time{}, fmt.Errorf("unrecognized VCS tool output: %v", err) + } + + return rev, time.Unix(secs, 0), nil +} + // vcsGit describes how to use Git. var vcsGit = &Cmd{ - Name: "Git", - Cmd: "git", + Name: "Git", + Cmd: "git", + RootNames: []string{".git"}, CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, @@ -182,6 +249,7 @@ var vcsGit = &Cmd{ PingCmd: "ls-remote {scheme}://{repo}", RemoteRepo: gitRemoteRepo, + Status: gitStatus, } // scpSyntaxRe matches the SCP-like addresses used by Git to access @@ -232,10 +300,40 @@ func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { return "", errParse } +func gitStatus(vcsGit *Cmd, rootDir string) (Status, error) { + out, err := vcsGit.runOutputVerboseOnly(rootDir, "status --porcelain") + if err != nil { + return Status{}, err + } + uncommitted := len(out) > 0 + + // "git status" works for empty repositories, but "git show" does not. + // Assume there are no commits in the repo when "git show" fails with + // uncommitted files and skip tagging revision / committime. + var rev string + var commitTime time.Time + out, err = vcsGit.runOutputVerboseOnly(rootDir, "show -s --no-show-signature --format=%H:%ct") + if err != nil && !uncommitted { + return Status{}, err + } else if err == nil { + rev, commitTime, err = parseRevTime(out) + if err != nil { + return Status{}, err + } + } + + return Status{ + Revision: rev, + CommitTime: commitTime, + Uncommitted: uncommitted, + }, nil +} + // vcsBzr describes how to use Bazaar. var vcsBzr = &Cmd{ - Name: "Bazaar", - Cmd: "bzr", + Name: "Bazaar", + Cmd: "bzr", + RootNames: []string{".bzr"}, CreateCmd: []string{"branch -- {repo} {dir}"}, @@ -251,6 +349,7 @@ var vcsBzr = &Cmd{ PingCmd: "info -- {scheme}://{repo}", RemoteRepo: bzrRemoteRepo, ResolveRepo: bzrResolveRepo, + Status: bzrStatus, } func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { @@ -294,10 +393,68 @@ func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, e return strings.TrimSpace(out), nil } +func bzrStatus(vcsBzr *Cmd, rootDir string) (Status, error) { + outb, err := vcsBzr.runOutputVerboseOnly(rootDir, "version-info") + if err != nil { + return Status{}, err + } + out := string(outb) + + // Expect (non-empty repositories only): + // + // revision-id: gopher@gopher.net-20211021072330-qshok76wfypw9lpm + // date: 2021-09-21 12:00:00 +1000 + // ... + var rev string + var commitTime time.Time + + for _, line := range strings.Split(out, "\n") { + i := strings.IndexByte(line, ':') + if i < 0 { + continue + } + key := line[:i] + value := strings.TrimSpace(line[i+1:]) + + switch key { + case "revision-id": + rev = value + case "date": + var err error + commitTime, err = time.Parse("2006-01-02 15:04:05 -0700", value) + if err != nil { + return Status{}, errors.New("unable to parse output of bzr version-info") + } + } + } + + outb, err = vcsBzr.runOutputVerboseOnly(rootDir, "status") + if err != nil { + return Status{}, err + } + + // Skip warning when working directory is set to an older revision. + if bytes.HasPrefix(outb, []byte("working tree is out of date")) { + i := bytes.IndexByte(outb, '\n') + if i < 0 { + i = len(outb) + } + outb = outb[:i] + } + uncommitted := len(outb) > 0 + + return Status{ + Revision: rev, + CommitTime: commitTime, + Uncommitted: uncommitted, + }, nil +} + // vcsSvn describes how to use Subversion. var vcsSvn = &Cmd{ - Name: "Subversion", - Cmd: "svn", + Name: "Subversion", + Cmd: "svn", + RootNames: []string{".svn"}, CreateCmd: []string{"checkout -- {repo} {dir}"}, DownloadCmd: []string{"update"}, @@ -346,8 +503,9 @@ const fossilRepoName = ".fossil" // vcsFossil describes how to use Fossil (fossil-scm.org) var vcsFossil = &Cmd{ - Name: "Fossil", - Cmd: "fossil", + Name: "Fossil", + Cmd: "fossil", + RootNames: []string{".fslckout", "_FOSSIL_"}, CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, DownloadCmd: []string{"up"}, @@ -358,6 +516,7 @@ var vcsFossil = &Cmd{ Scheme: []string{"https", "http"}, RemoteRepo: fossilRemoteRepo, + Status: fossilStatus, } func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { @@ -368,6 +527,60 @@ func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err er return strings.TrimSpace(string(out)), nil } +var errFossilInfo = errors.New("unable to parse output of fossil info") + +func fossilStatus(vcsFossil *Cmd, rootDir string) (Status, error) { + outb, err := vcsFossil.runOutputVerboseOnly(rootDir, "info") + if err != nil { + return Status{}, err + } + out := string(outb) + + // Expect: + // ... + // checkout: 91ed71f22c77be0c3e250920f47bfd4e1f9024d2 2021-09-21 12:00:00 UTC + // ... + + // Extract revision and commit time. + // Ensure line ends with UTC (known timezone offset). + const prefix = "\ncheckout:" + const suffix = " UTC" + i := strings.Index(out, prefix) + if i < 0 { + return Status{}, errFossilInfo + } + checkout := out[i+len(prefix):] + i = strings.Index(checkout, suffix) + if i < 0 { + return Status{}, errFossilInfo + } + checkout = strings.TrimSpace(checkout[:i]) + + i = strings.IndexByte(checkout, ' ') + if i < 0 { + return Status{}, errFossilInfo + } + rev := checkout[:i] + + commitTime, err := time.ParseInLocation("2006-01-02 15:04:05", checkout[i+1:], time.UTC) + if err != nil { + return Status{}, fmt.Errorf("%v: %v", errFossilInfo, err) + } + + // Also look for untracked changes. + outb, err = vcsFossil.runOutputVerboseOnly(rootDir, "changes --differ") + if err != nil { + return Status{}, err + } + uncommitted := len(outb) > 0 + + return Status{ + Revision: rev, + CommitTime: commitTime, + Uncommitted: uncommitted, + }, nil +} + func (v *Cmd) String() string { return v.Name } @@ -395,6 +608,12 @@ func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error return v.run1(dir, cmd, keyval, true) } +// runOutputVerboseOnly is like runOutput but only generates error output to +// standard error in verbose mode. +func (v *Cmd) runOutputVerboseOnly(dir string, cmd string, keyval ...string) ([]byte, error) { + return v.run1(dir, cmd, keyval, false) +} + // run1 is the generalized implementation of run and runOutput. func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { m := make(map[string]string) @@ -550,58 +769,86 @@ type vcsPath struct { // FromDir inspects dir and its parents to determine the // version control system and code repository to use. -// On return, root is the import path -// corresponding to the root of the repository. -func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { +// If no repository is found, FromDir returns an error +// equivalent to os.ErrNotExist. +func FromDir(dir, srcRoot string, allowNesting bool) (repoDir string, vcsCmd *Cmd, err error) { // Clean and double-check that dir is in (a subdirectory of) srcRoot. dir = filepath.Clean(dir) - srcRoot = filepath.Clean(srcRoot) - if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { - return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) + if srcRoot != "" { + srcRoot = filepath.Clean(srcRoot) + if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { + return "", nil, fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) + } } - var vcsRet *Cmd - var rootRet string - origDir := dir for len(dir) > len(srcRoot) { for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { - root := filepath.ToSlash(dir[len(srcRoot)+1:]) - // Record first VCS we find, but keep looking, - // to detect mistakes like one kind of VCS inside another. - if vcsRet == nil { - vcsRet = vcs - rootRet = root + if _, err := statAny(dir, vcs.RootNames); err == nil { + // Record first VCS we find. + // If allowNesting is false (as it is in GOPATH), keep looking for + // repositories in parent directories and report an error if one is + // found to mitigate VCS injection attacks. + if vcsCmd == nil { + vcsCmd = vcs + repoDir = dir + if allowNesting { + return repoDir, vcsCmd, nil + } continue } // Allow .git inside .git, which can arise due to submodules. - if vcsRet == vcs && vcs.Cmd == "git" { + if vcsCmd == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. - return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", - filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) + return "", nil, fmt.Errorf("directory %q uses %s, but parent %q uses %s", + repoDir, vcsCmd.Cmd, dir, vcs.Cmd) } } // Move to parent. ndir := filepath.Dir(dir) if len(ndir) >= len(dir) { - // Shouldn't happen, but just in case, stop. break } dir = ndir } + if vcsCmd == nil { + return "", nil, &vcsNotFoundError{dir: origDir} + } + return repoDir, vcsCmd, nil +} - if vcsRet != nil { - if err := checkGOVCS(vcsRet, rootRet); err != nil { - return nil, "", err +// statAny provides FileInfo for the first filename found in the directory. +// Otherwise, it returns the last error seen. +func statAny(dir string, filenames []string) (os.FileInfo, error) { + if len(filenames) == 0 { + return nil, errors.New("invalid argument: no filenames provided") + } + + var err error + var fi os.FileInfo + for _, name := range filenames { + fi, err = os.Stat(filepath.Join(dir, name)) + if err == nil { + return fi, nil } - return vcsRet, rootRet, nil } - return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) + return nil, err +} + +type vcsNotFoundError struct { + dir string +} + +func (e *vcsNotFoundError) Error() string { + return fmt.Sprintf("directory %q is not using a known version control system", e.dir) +} + +func (e *vcsNotFoundError) Is(err error) bool { + return err == os.ErrNotExist } // A govcsRule is a single GOVCS rule like private:hg|svn. @@ -707,7 +954,11 @@ var defaultGOVCS = govcsConfig{ {"public", []string{"git", "hg"}}, } -func checkGOVCS(vcs *Cmd, root string) error { +// CheckGOVCS checks whether the policy defined by the environment variable +// GOVCS allows the given vcs command to be used with the given repository +// root path. Note that root may not be a real package or module path; it's +// the same as the root path in the go-import meta tag. +func CheckGOVCS(vcs *Cmd, root string) error { if vcs == vcsMod { // Direct module (proxy protocol) fetches don't // involve an external version control system @@ -745,7 +996,7 @@ func CheckNested(vcs *Cmd, dir, srcRoot string) error { otherDir := dir for len(otherDir) > len(srcRoot) { for _, otherVCS := range vcsList { - if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { + if _, err := statAny(otherDir, otherVCS.RootNames); err == nil { // Allow expected vcs in original dir. if otherDir == dir && otherVCS == vcs { continue @@ -885,7 +1136,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if vcs == nil { return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) } - if err := checkGOVCS(vcs, match["root"]); err != nil { + if err := CheckGOVCS(vcs, match["root"]); err != nil { return nil, err } var repoURL string @@ -1012,7 +1263,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se } } - if err := checkGOVCS(vcs, mmi.Prefix); err != nil { + if err := CheckGOVCS(vcs, mmi.Prefix); err != nil { return nil, err } @@ -1063,7 +1314,7 @@ func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.Secu return res, nil } - resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { + resi, _, _ := fetchGroup.Do(importPrefix, func() (resi any, err error) { fetchCacheMu.Lock() if res, ok := fetchCache[importPrefix]; ok { fetchCacheMu.Unlock() @@ -1189,8 +1440,9 @@ var vcsPaths = []*vcsPath{ { pathPrefix: "bitbucket.org", regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`), + vcs: "git", repo: "https://{root}", - check: bitbucketVCS, + check: noVCSSuffix, }, // IBM DevOps Services (JazzHub) @@ -1262,56 +1514,6 @@ func noVCSSuffix(match map[string]string) error { return nil } -// bitbucketVCS determines the version control system for a -// Bitbucket repository, by using the Bitbucket API. -func bitbucketVCS(match map[string]string) error { - if err := noVCSSuffix(match); err != nil { - return err - } - - var resp struct { - SCM string `json:"scm"` - } - url := &urlpkg.URL{ - Scheme: "https", - Host: "api.bitbucket.org", - Path: expand(match, "/2.0/repositories/{bitname}"), - RawQuery: "fields=scm", - } - data, err := web.GetBytes(url) - if err != nil { - if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { - // this may be a private repository. If so, attempt to determine which - // VCS it uses. See issue 5375. - root := match["root"] - for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).Ping("https", root) == nil { - resp.SCM = vcs - break - } - } - } - - if resp.SCM == "" { - return err - } - } else { - if err := json.Unmarshal(data, &resp); err != nil { - return fmt.Errorf("decoding %s: %v", url, err) - } - } - - if vcsByCmd(resp.SCM) != nil { - match["vcs"] = resp.SCM - if resp.SCM == "git" { - match["repo"] += ".git" - } - return nil - } - - return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") -} - // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, // "foo" could be a series name registered in Launchpad with its own branch, // and it could also be the name of a directory within the main project @@ -1340,7 +1542,7 @@ type importError struct { err error } -func importErrorf(path, format string, args ...interface{}) error { +func importErrorf(path, format string, args ...any) error { err := &importError{importPath: path, err: fmt.Errorf(format, args...)} if errStr := err.Error(); !strings.Contains(errStr, path) { panic(fmt.Sprintf("path %q not in error %q", path, errStr)) diff --git a/libgo/go/cmd/go/internal/vcs/vcs_test.go b/libgo/go/cmd/go/internal/vcs/vcs_test.go index c5c7a3283bc..943d520d547 100644 --- a/libgo/go/cmd/go/internal/vcs/vcs_test.go +++ b/libgo/go/cmd/go/internal/vcs/vcs_test.go @@ -6,9 +6,9 @@ package vcs import ( "errors" + "fmt" "internal/testenv" "os" - "path" "path/filepath" "strings" "testing" @@ -183,6 +183,13 @@ func TestRepoRootForImportPath(t *testing.T) { "chiselapp.com/user/kyle/fossilgg", nil, }, + { + "bitbucket.org/workspace/pkgname", + &RepoRoot{ + VCS: vcsGit, + Repo: "https://bitbucket.org/workspace/pkgname", + }, + }, } for _, test := range tests { @@ -205,7 +212,8 @@ func TestRepoRootForImportPath(t *testing.T) { } } -// Test that vcsFromDir correctly inspects a given directory and returns the right VCS and root. +// Test that vcs.FromDir correctly inspects a given directory and returns the +// right VCS and repo directory. func TestFromDir(t *testing.T) { tempDir, err := os.MkdirTemp("", "vcstest") if err != nil { @@ -214,36 +222,35 @@ func TestFromDir(t *testing.T) { defer os.RemoveAll(tempDir) for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.Name, "."+vcs.Cmd) - if j&1 == 0 { - err := os.MkdirAll(dir, 0755) - if err != nil { - t.Fatal(err) + for r, rootName := range vcs.RootNames { + vcsName := fmt.Sprint(vcs.Name, r) + dir := filepath.Join(tempDir, "example.com", vcsName, rootName) + if j&1 == 0 { + err := os.MkdirAll(dir, 0755) + if err != nil { + t.Fatal(err) + } + } else { + err := os.MkdirAll(filepath.Dir(dir), 0755) + if err != nil { + t.Fatal(err) + } + f, err := os.Create(dir) + if err != nil { + t.Fatal(err) + } + f.Close() } - } else { - err := os.MkdirAll(filepath.Dir(dir), 0755) + + wantRepoDir := filepath.Dir(dir) + gotRepoDir, gotVCS, err := FromDir(dir, tempDir, false) if err != nil { - t.Fatal(err) + t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) + continue } - f, err := os.Create(dir) - if err != nil { - t.Fatal(err) + if gotRepoDir != wantRepoDir || gotVCS.Name != vcs.Name { + t.Errorf("FromDir(%q, %q) = RepoDir(%s), VCS(%s); want RepoDir(%s), VCS(%s)", dir, tempDir, gotRepoDir, gotVCS.Name, wantRepoDir, vcs.Name) } - f.Close() - } - - want := RepoRoot{ - VCS: vcs, - Root: path.Join("example.com", vcs.Name), - } - var got RepoRoot - got.VCS, got.Root, err = FromDir(dir, tempDir) - if err != nil { - t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) - continue - } - if got.VCS.Name != want.VCS.Name || got.Root != want.Root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.VCS, got.Root, want.VCS, want.Root) } } } diff --git a/libgo/go/cmd/go/internal/version/exe.go b/libgo/go/cmd/go/internal/version/exe.go deleted file mode 100644 index 0e7deef1491..00000000000 --- a/libgo/go/cmd/go/internal/version/exe.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package version - -import ( - "bytes" - "debug/elf" - "debug/macho" - "debug/pe" - "fmt" - "internal/xcoff" - "io" - "os" -) - -// An exe is a generic interface to an OS executable (ELF, Mach-O, PE, XCOFF). -type exe interface { - // Close closes the underlying file. - Close() error - - // ReadData reads and returns up to size byte starting at virtual address addr. - ReadData(addr, size uint64) ([]byte, error) - - // DataStart returns the writable data segment start address. - DataStart() uint64 -} - -// openExe opens file and returns it as an exe. -func openExe(file string) (exe, error) { - f, err := os.Open(file) - if err != nil { - return nil, err - } - data := make([]byte, 16) - if _, err := io.ReadFull(f, data); err != nil { - return nil, err - } - f.Seek(0, 0) - if bytes.HasPrefix(data, []byte("\x7FELF")) { - e, err := elf.NewFile(f) - if err != nil { - f.Close() - return nil, err - } - return &elfExe{f, e}, nil - } - if bytes.HasPrefix(data, []byte("MZ")) { - e, err := pe.NewFile(f) - if err != nil { - f.Close() - return nil, err - } - return &peExe{f, e}, nil - } - if bytes.HasPrefix(data, []byte("\xFE\xED\xFA")) || bytes.HasPrefix(data[1:], []byte("\xFA\xED\xFE")) { - e, err := macho.NewFile(f) - if err != nil { - f.Close() - return nil, err - } - return &machoExe{f, e}, nil - } - if bytes.HasPrefix(data, []byte{0x01, 0xDF}) || bytes.HasPrefix(data, []byte{0x01, 0xF7}) { - e, err := xcoff.NewFile(f) - if err != nil { - f.Close() - return nil, err - } - return &xcoffExe{f, e}, nil - - } - return nil, fmt.Errorf("unrecognized executable format") -} - -// elfExe is the ELF implementation of the exe interface. -type elfExe struct { - os *os.File - f *elf.File -} - -func (x *elfExe) Close() error { - return x.os.Close() -} - -func (x *elfExe) ReadData(addr, size uint64) ([]byte, error) { - for _, prog := range x.f.Progs { - if prog.Vaddr <= addr && addr <= prog.Vaddr+prog.Filesz-1 { - n := prog.Vaddr + prog.Filesz - addr - if n > size { - n = size - } - data := make([]byte, n) - _, err := prog.ReadAt(data, int64(addr-prog.Vaddr)) - if err != nil { - return nil, err - } - return data, nil - } - } - return nil, fmt.Errorf("address not mapped") -} - -func (x *elfExe) DataStart() uint64 { - for _, s := range x.f.Sections { - if s.Name == ".go.buildinfo" { - return s.Addr - } - } - for _, p := range x.f.Progs { - if p.Type == elf.PT_LOAD && p.Flags&(elf.PF_X|elf.PF_W) == elf.PF_W { - return p.Vaddr - } - } - return 0 -} - -// peExe is the PE (Windows Portable Executable) implementation of the exe interface. -type peExe struct { - os *os.File - f *pe.File -} - -func (x *peExe) Close() error { - return x.os.Close() -} - -func (x *peExe) imageBase() uint64 { - switch oh := x.f.OptionalHeader.(type) { - case *pe.OptionalHeader32: - return uint64(oh.ImageBase) - case *pe.OptionalHeader64: - return oh.ImageBase - } - return 0 -} - -func (x *peExe) ReadData(addr, size uint64) ([]byte, error) { - addr -= x.imageBase() - for _, sect := range x.f.Sections { - if uint64(sect.VirtualAddress) <= addr && addr <= uint64(sect.VirtualAddress+sect.Size-1) { - n := uint64(sect.VirtualAddress+sect.Size) - addr - if n > size { - n = size - } - data := make([]byte, n) - _, err := sect.ReadAt(data, int64(addr-uint64(sect.VirtualAddress))) - if err != nil { - return nil, err - } - return data, nil - } - } - return nil, fmt.Errorf("address not mapped") -} - -func (x *peExe) DataStart() uint64 { - // Assume data is first writable section. - const ( - IMAGE_SCN_CNT_CODE = 0x00000020 - IMAGE_SCN_CNT_INITIALIZED_DATA = 0x00000040 - IMAGE_SCN_CNT_UNINITIALIZED_DATA = 0x00000080 - IMAGE_SCN_MEM_EXECUTE = 0x20000000 - IMAGE_SCN_MEM_READ = 0x40000000 - IMAGE_SCN_MEM_WRITE = 0x80000000 - IMAGE_SCN_MEM_DISCARDABLE = 0x2000000 - IMAGE_SCN_LNK_NRELOC_OVFL = 0x1000000 - IMAGE_SCN_ALIGN_32BYTES = 0x600000 - ) - for _, sect := range x.f.Sections { - if sect.VirtualAddress != 0 && sect.Size != 0 && - sect.Characteristics&^IMAGE_SCN_ALIGN_32BYTES == IMAGE_SCN_CNT_INITIALIZED_DATA|IMAGE_SCN_MEM_READ|IMAGE_SCN_MEM_WRITE { - return uint64(sect.VirtualAddress) + x.imageBase() - } - } - return 0 -} - -// machoExe is the Mach-O (Apple macOS/iOS) implementation of the exe interface. -type machoExe struct { - os *os.File - f *macho.File -} - -func (x *machoExe) Close() error { - return x.os.Close() -} - -func (x *machoExe) ReadData(addr, size uint64) ([]byte, error) { - for _, load := range x.f.Loads { - seg, ok := load.(*macho.Segment) - if !ok { - continue - } - if seg.Addr <= addr && addr <= seg.Addr+seg.Filesz-1 { - if seg.Name == "__PAGEZERO" { - continue - } - n := seg.Addr + seg.Filesz - addr - if n > size { - n = size - } - data := make([]byte, n) - _, err := seg.ReadAt(data, int64(addr-seg.Addr)) - if err != nil { - return nil, err - } - return data, nil - } - } - return nil, fmt.Errorf("address not mapped") -} - -func (x *machoExe) DataStart() uint64 { - // Look for section named "__go_buildinfo". - for _, sec := range x.f.Sections { - if sec.Name == "__go_buildinfo" { - return sec.Addr - } - } - // Try the first non-empty writable segment. - const RW = 3 - for _, load := range x.f.Loads { - seg, ok := load.(*macho.Segment) - if ok && seg.Addr != 0 && seg.Filesz != 0 && seg.Prot == RW && seg.Maxprot == RW { - return seg.Addr - } - } - return 0 -} - -// xcoffExe is the XCOFF (AIX eXtended COFF) implementation of the exe interface. -type xcoffExe struct { - os *os.File - f *xcoff.File -} - -func (x *xcoffExe) Close() error { - return x.os.Close() -} - -func (x *xcoffExe) ReadData(addr, size uint64) ([]byte, error) { - for _, sect := range x.f.Sections { - if uint64(sect.VirtualAddress) <= addr && addr <= uint64(sect.VirtualAddress+sect.Size-1) { - n := uint64(sect.VirtualAddress+sect.Size) - addr - if n > size { - n = size - } - data := make([]byte, n) - _, err := sect.ReadAt(data, int64(addr-uint64(sect.VirtualAddress))) - if err != nil { - return nil, err - } - return data, nil - } - } - return nil, fmt.Errorf("address not mapped") -} - -func (x *xcoffExe) DataStart() uint64 { - return x.f.SectionByType(xcoff.STYP_DATA).VirtualAddress -} diff --git a/libgo/go/cmd/go/internal/version/version.go b/libgo/go/cmd/go/internal/version/version.go index 58cbd32e78d..52502e95c6d 100644 --- a/libgo/go/cmd/go/internal/version/version.go +++ b/libgo/go/cmd/go/internal/version/version.go @@ -8,7 +8,8 @@ package version import ( "bytes" "context" - "encoding/binary" + "debug/buildinfo" + "errors" "fmt" "io/fs" "os" @@ -62,8 +63,14 @@ func runVersion(ctx context.Context, cmd *base.Command, args []string) { // a reasonable use case. For example, imagine GOFLAGS=-v to // turn "verbose mode" on for all Go commands, which should not // break "go version". - if (!base.InGOFLAGS("-m") && *versionM) || (!base.InGOFLAGS("-v") && *versionV) { - fmt.Fprintf(os.Stderr, "go version: flags can only be used with arguments\n") + var argOnlyFlag string + if !base.InGOFLAGS("-m") && *versionM { + argOnlyFlag = "-m" + } else if !base.InGOFLAGS("-v") && *versionV { + argOnlyFlag = "-v" + } + if argOnlyFlag != "" { + fmt.Fprintf(os.Stderr, "go: 'go version' only accepts %s flag with arguments\n", argOnlyFlag) base.SetExitStatus(2) return } @@ -135,90 +142,26 @@ func scanFile(file string, info fs.FileInfo, mustPrint bool) { return } - x, err := openExe(file) + bi, err := buildinfo.ReadFile(file) if err != nil { if mustPrint { - fmt.Fprintf(os.Stderr, "%s: %v\n", file, err) - } - return - } - defer x.Close() - - vers, mod := findVers(x) - if vers == "" { - if mustPrint { - fmt.Fprintf(os.Stderr, "%s: go version not found\n", file) + if pathErr := (*os.PathError)(nil); errors.As(err, &pathErr) && filepath.Clean(pathErr.Path) == filepath.Clean(file) { + fmt.Fprintf(os.Stderr, "%v\n", file) + } else { + fmt.Fprintf(os.Stderr, "%s: %v\n", file, err) + } } return } - fmt.Printf("%s: %s\n", file, vers) - if *versionM && mod != "" { - fmt.Printf("\t%s\n", strings.ReplaceAll(mod[:len(mod)-1], "\n", "\n\t")) - } -} - -// The build info blob left by the linker is identified by -// a 16-byte header, consisting of buildInfoMagic (14 bytes), -// the binary's pointer size (1 byte), -// and whether the binary is big endian (1 byte). -var buildInfoMagic = []byte("\xff Go buildinf:") - -// findVers finds and returns the Go version and module version information -// in the executable x. -func findVers(x exe) (vers, mod string) { - // Read the first 64kB of text to find the build info blob. - text := x.DataStart() - data, err := x.ReadData(text, 64*1024) + fmt.Printf("%s: %s\n", file, bi.GoVersion) + bi.GoVersion = "" // suppress printing go version again + mod, err := bi.MarshalText() if err != nil { + fmt.Fprintf(os.Stderr, "%s: formatting build info: %v\n", file, err) return } - for ; !bytes.HasPrefix(data, buildInfoMagic); data = data[32:] { - if len(data) < 32 { - return - } - } - - // Decode the blob. - ptrSize := int(data[14]) - bigEndian := data[15] != 0 - var bo binary.ByteOrder - if bigEndian { - bo = binary.BigEndian - } else { - bo = binary.LittleEndian - } - var readPtr func([]byte) uint64 - if ptrSize == 4 { - readPtr = func(b []byte) uint64 { return uint64(bo.Uint32(b)) } - } else { - readPtr = bo.Uint64 - } - vers = readString(x, ptrSize, readPtr, readPtr(data[16:])) - if vers == "" { - return - } - mod = readString(x, ptrSize, readPtr, readPtr(data[16+ptrSize:])) - if len(mod) >= 33 && mod[len(mod)-17] == '\n' { - // Strip module framing. - mod = mod[16 : len(mod)-16] - } else { - mod = "" - } - return -} - -// readString returns the string at address addr in the executable x. -func readString(x exe, ptrSize int, readPtr func([]byte) uint64, addr uint64) string { - hdr, err := x.ReadData(addr, uint64(2*ptrSize)) - if err != nil || len(hdr) < 2*ptrSize { - return "" - } - dataAddr := readPtr(hdr) - dataLen := readPtr(hdr[ptrSize:]) - data, err := x.ReadData(dataAddr, dataLen) - if err != nil || uint64(len(data)) < dataLen { - return "" + if *versionM && len(mod) > 0 { + fmt.Printf("\t%s\n", bytes.ReplaceAll(mod[:len(mod)-1], []byte("\n"), []byte("\n\t"))) } - return string(data) } diff --git a/libgo/go/cmd/go/internal/vet/vet.go b/libgo/go/cmd/go/internal/vet/vet.go index 1d419dddb98..88b3c570a03 100644 --- a/libgo/go/cmd/go/internal/vet/vet.go +++ b/libgo/go/cmd/go/internal/vet/vet.go @@ -103,7 +103,7 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { continue } if len(ptest.GoFiles) == 0 && len(ptest.CgoFiles) == 0 && pxtest == nil { - base.Errorf("go vet %s: no Go files in %s", p.ImportPath, p.Dir) + base.Errorf("go: can't vet %s: no Go files in %s", p.ImportPath, p.Dir) continue } if len(ptest.GoFiles) > 0 || len(ptest.CgoFiles) > 0 { diff --git a/libgo/go/cmd/go/internal/vet/vetflag.go b/libgo/go/cmd/go/internal/vet/vetflag.go index b5b3c462ff2..3551a5997c5 100644 --- a/libgo/go/cmd/go/internal/vet/vetflag.go +++ b/libgo/go/cmd/go/internal/vet/vetflag.go @@ -82,7 +82,7 @@ func vetFlags(args []string) (passToVet, packageNames []string) { vetcmd := exec.Command(tool, "-flags") vetcmd.Stdout = out if err := vetcmd.Run(); err != nil { - fmt.Fprintf(os.Stderr, "go vet: can't execute %s -flags: %v\n", tool, err) + fmt.Fprintf(os.Stderr, "go: can't execute %s -flags: %v\n", tool, err) base.SetExitStatus(2) base.Exit() } @@ -92,7 +92,7 @@ func vetFlags(args []string) (passToVet, packageNames []string) { Usage string } if err := json.Unmarshal(out.Bytes(), &analysisFlags); err != nil { - fmt.Fprintf(os.Stderr, "go vet: can't unmarshal JSON from %s -flags: %v", tool, err) + fmt.Fprintf(os.Stderr, "go: can't unmarshal JSON from %s -flags: %v", tool, err) base.SetExitStatus(2) base.Exit() } diff --git a/libgo/go/cmd/go/internal/web/bootstrap.go b/libgo/go/cmd/go/internal/web/bootstrap.go index 08686cdfcf9..ab88e9e4781 100644 --- a/libgo/go/cmd/go/internal/web/bootstrap.go +++ b/libgo/go/cmd/go/internal/web/bootstrap.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build cmd_go_bootstrap -// +build cmd_go_bootstrap // This code is compiled only into the bootstrap 'go' binary. // These stubs avoid importing packages with large dependency diff --git a/libgo/go/cmd/go/internal/web/http.go b/libgo/go/cmd/go/internal/web/http.go index f177278eba1..a92326db01e 100644 --- a/libgo/go/cmd/go/internal/web/http.go +++ b/libgo/go/cmd/go/internal/web/http.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !cmd_go_bootstrap -// +build !cmd_go_bootstrap // This code is compiled into the real 'go' binary, but it is not // compiled into the binary that is built during all.bash, so as @@ -17,6 +16,7 @@ import ( "errors" "fmt" "mime" + "net" "net/http" urlpkg "net/url" "os" @@ -84,8 +84,15 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { if url.Host == "localhost.localdev" { return nil, fmt.Errorf("no such host localhost.localdev") } - if os.Getenv("TESTGONETWORK") == "panic" && !strings.HasPrefix(url.Host, "127.0.0.1") && !strings.HasPrefix(url.Host, "0.0.0.0") { - panic("use of network: " + url.String()) + if os.Getenv("TESTGONETWORK") == "panic" { + host := url.Host + if h, _, err := net.SplitHostPort(url.Host); err == nil && h != "" { + host = h + } + addr := net.ParseIP(host) + if addr == nil || (!addr.IsLoopback() && !addr.IsUnspecified()) { + panic("use of network: " + url.String()) + } } fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) { diff --git a/libgo/go/cmd/go/internal/web/url_other.go b/libgo/go/cmd/go/internal/web/url_other.go index 453af402b43..84bbd72820f 100644 --- a/libgo/go/cmd/go/internal/web/url_other.go +++ b/libgo/go/cmd/go/internal/web/url_other.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !windows -// +build !windows package web diff --git a/libgo/go/cmd/go/internal/web/url_other_test.go b/libgo/go/cmd/go/internal/web/url_other_test.go index 4d6ed2ec7f8..5c197de800d 100644 --- a/libgo/go/cmd/go/internal/web/url_other_test.go +++ b/libgo/go/cmd/go/internal/web/url_other_test.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !windows -// +build !windows package web diff --git a/libgo/go/cmd/go/internal/work/action.go b/libgo/go/cmd/go/internal/work/action.go index 2e00c0cb126..ba82e8ff225 100644 --- a/libgo/go/cmd/go/internal/work/action.go +++ b/libgo/go/cmd/go/internal/work/action.go @@ -38,7 +38,7 @@ type Builder struct { actionCache map[cacheKey]*Action // a cache of already-constructed actions mkdirCache map[string]bool // a cache of created directories flagCache map[[2]string]bool // a cache of supported compiler flags - Print func(args ...interface{}) (int, error) + Print func(args ...any) (int, error) IsCmdList bool // running as part of go list; set p.Stale and additional fields below NeedError bool // list needs p.Error @@ -121,8 +121,8 @@ type actionQueue []*Action func (q *actionQueue) Len() int { return len(*q) } func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } -func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*Action)) } -func (q *actionQueue) Pop() interface{} { +func (q *actionQueue) Push(x any) { *q = append(*q, x.(*Action)) } +func (q *actionQueue) Pop() any { n := len(*q) - 1 x := (*q)[n] *q = (*q)[:n] @@ -242,7 +242,7 @@ const ( ) func (b *Builder) Init() { - b.Print = func(a ...interface{}) (int, error) { + b.Print = func(a ...any) (int, error) { return fmt.Fprint(os.Stderr, a...) } b.actionCache = make(map[cacheKey]*Action) @@ -295,14 +295,14 @@ func (b *Builder) Init() { } if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil { - fmt.Fprintf(os.Stderr, "cmd/go: %v\n", err) + fmt.Fprintf(os.Stderr, "go: %v\n", err) base.SetExitStatus(2) base.Exit() } for _, tag := range cfg.BuildContext.BuildTags { if strings.Contains(tag, ",") { - fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") + fmt.Fprintf(os.Stderr, "go: -tags space-separated list contains comma\n") base.SetExitStatus(2) base.Exit() } diff --git a/libgo/go/cmd/go/internal/work/build.go b/libgo/go/cmd/go/internal/work/build.go index 0ed2389cd5a..1c278d3d992 100644 --- a/libgo/go/cmd/go/internal/work/build.go +++ b/libgo/go/cmd/go/internal/work/build.go @@ -68,13 +68,16 @@ and test commands: The default is GOMAXPROCS, normally the number of CPUs available. -race enable data race detection. - Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64, + Supported only on linux/amd64, freebsd/amd64, darwin/amd64, darwin/arm64, windows/amd64, linux/ppc64le and linux/arm64 (only for 48-bit VMA). -msan enable interoperation with memory sanitizer. Supported only on linux/amd64, linux/arm64 and only with Clang/LLVM as the host C compiler. On linux/arm64, pie build mode will be used. + -asan + enable interoperation with address sanitizer. + Supported only on linux/arm64, linux/amd64. -v print the names of packages as they are compiled. -work @@ -87,6 +90,12 @@ and test commands: arguments to pass on each go tool asm invocation. -buildmode mode build mode to use. See 'go help buildmode' for more. + -buildvcs + Whether to stamp binaries with version control information. By default, + version control information is stamped into a binary if the main package + and the main module containing it are in the repository containing the + current directory (if there is a repository). Use -buildvcs=false to + omit version control information. -compiler name name of compiler to use, as in runtime.Compiler (gccgo or gc). -gccgoflags '[pattern=]arg list' @@ -98,8 +107,8 @@ and test commands: in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to race or, if set explicitly, has _race appended to it. Likewise for the -msan - flag. Using a -buildmode option that requires non-default compile flags - has a similar effect. + and -asan flags. Using a -buildmode option that requires non-default compile + flags has a similar effect. -ldflags '[pattern=]arg list' arguments to pass on each go tool link invocation. -linkshared @@ -121,6 +130,14 @@ and test commands: directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum". + -workfile file + in module aware mode, use the given go.work file as a workspace file. + By default or when -workfile is "auto", the go command searches for a + file named go.work in the current directory and then containing directories + until one is found. If a valid go.work file is found, the modules + specified will collectively be used as the main modules. If -workfile + is "off", or a go.work file is not found in "auto" mode, workspace + mode is disabled. -overlay file read a JSON config file that provides an overlay for build operations. The file is a JSON struct with a single field, named 'Replace', that @@ -145,9 +162,8 @@ and test commands: -trimpath remove all file system paths from the resulting executable. Instead of absolute file system paths, the recorded file names - will begin with either "go" (for the standard library), - or a module path@version (when using modules), - or a plain import path (when using GOPATH). + will begin either a module path@version (when using modules), + or a plain import path (when using the standard library, or GOPATH). -toolexec 'cmd args' a program to use to invoke toolchain programs like vet and asm. For example, instead of running asm, the go command will run @@ -201,6 +217,7 @@ func init() { AddBuildFlags(CmdBuild, DefaultBuildFlags) AddBuildFlags(CmdInstall, DefaultBuildFlags) + base.AddWorkfileFlag(&CmdBuild.Flag) } // Note that flags consulted by other parts of the code @@ -289,10 +306,12 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.StringVar(&cfg.BuildPkgdir, "pkgdir", "", "") cmd.Flag.BoolVar(&cfg.BuildRace, "race", false, "") cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") + cmd.Flag.BoolVar(&cfg.BuildASan, "asan", false, "") cmd.Flag.Var((*tagsFlag)(&cfg.BuildContext.BuildTags), "tags", "") cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildToolexec), "toolexec", "") cmd.Flag.BoolVar(&cfg.BuildTrimpath, "trimpath", false, "") cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") + cmd.Flag.BoolVar(&cfg.BuildBuildvcs, "buildvcs", true, "") // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") @@ -364,6 +383,7 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } var runtimeVersion = runtime.Version() func runBuild(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() BuildInit() var b Builder b.Init() @@ -396,7 +416,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { depMode := ModeBuild if cfg.BuildI { depMode = ModeInstall - fmt.Fprint(os.Stderr, "go build: -i flag is deprecated\n") + fmt.Fprint(os.Stderr, "go: -i flag is deprecated\n") } pkgs = omitTestOnly(pkgsFilter(pkgs)) @@ -415,7 +435,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { strings.HasSuffix(cfg.BuildO, "/") || strings.HasSuffix(cfg.BuildO, string(os.PathSeparator)) { if !explicitO { - base.Fatalf("go build: build output %q already exists and is a directory", cfg.BuildO) + base.Fatalf("go: build output %q already exists and is a directory", cfg.BuildO) } a := &Action{Mode: "go build"} for _, p := range pkgs { @@ -430,13 +450,13 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { a.Deps = append(a.Deps, b.AutoAction(ModeInstall, depMode, p)) } if len(a.Deps) == 0 { - base.Fatalf("go build: no main packages to build") + base.Fatalf("go: no main packages to build") } b.Do(ctx, a) return } if len(pkgs) > 1 { - base.Fatalf("go build: cannot write multiple packages to non-directory %s", cfg.BuildO) + base.Fatalf("go: cannot write multiple packages to non-directory %s", cfg.BuildO) } else if len(pkgs) == 0 { base.Fatalf("no packages to build") } @@ -486,14 +506,17 @@ allowed, even if they refer to the same version. - All arguments must refer to packages in the same module at the same version. +- Package path arguments must refer to main packages. Pattern arguments +will only match main packages. + - No module is considered the "main" module. If the module containing packages named on the command line has a go.mod file, it must not contain directives (replace and exclude) that would cause it to be interpreted differently than if it were the main module. The module must not require a higher version of itself. -- Package path arguments must refer to main packages. Pattern arguments -will only match main packages. +- Vendor directories are not used in any module. (Vendor directories are not +included in the module zip files downloaded by 'go install'.) If the arguments don't have version suffixes, "go install" may run in module-aware mode or GOPATH mode, depending on the GO111MODULE environment @@ -580,13 +603,14 @@ func runInstall(ctx context.Context, cmd *base.Command, args []string) { for _, arg := range args { if strings.Contains(arg, "@") && !build.IsLocalImport(arg) && !filepath.IsAbs(arg) { if cfg.BuildI { - fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + fmt.Fprint(os.Stderr, "go: -i flag is deprecated\n") } installOutsideModule(ctx, args) return } } + modload.InitWorkfile() BuildInit() pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{}, args) if cfg.ModulesEnabled && !modload.HasModRoot() { @@ -608,7 +632,7 @@ func runInstall(ctx context.Context, cmd *base.Command, args []string) { latestArgs[i] = args[i] + "@latest" } hint := strings.Join(latestArgs, " ") - base.Fatalf("go install: version is required when current directory is not in a module\n\tTry 'go install %s' to install the latest version", hint) + base.Fatalf("go: 'go install' requires a version when current directory is not in a module\n\tTry 'go install %s' to install the latest version", hint) } } load.CheckPackageErrors(pkgs) @@ -621,7 +645,7 @@ func runInstall(ctx context.Context, cmd *base.Command, args []string) { } } if !allGoroot { - fmt.Fprint(os.Stderr, "go install: -i flag is deprecated\n") + fmt.Fprintf(os.Stderr, "go: -i flag is deprecated\n") } } @@ -667,14 +691,14 @@ func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Packag case p.Name != "main" && p.Module != nil: // Non-executables have no target (except the cache) when building with modules. case p.Internal.GobinSubdir: - base.Errorf("go %s: cannot install cross-compiled binaries when GOBIN is set", cfg.CmdName) + base.Errorf("go: cannot install cross-compiled binaries when GOBIN is set") case p.Internal.CmdlineFiles: - base.Errorf("go %s: no install location for .go files listed on command line (GOBIN not set)", cfg.CmdName) + base.Errorf("go: no install location for .go files listed on command line (GOBIN not set)") case p.ConflictDir != "": - base.Errorf("go %s: no install location for %s: hidden by %s", cfg.CmdName, p.Dir, p.ConflictDir) + base.Errorf("go: no install location for %s: hidden by %s", p.Dir, p.ConflictDir) default: - base.Errorf("go %s: no install location for directory %s outside GOPATH\n"+ - "\tFor more details see: 'go help gopath'", cfg.CmdName, p.Dir) + base.Errorf("go: no install location for directory %s outside GOPATH\n"+ + "\tFor more details see: 'go help gopath'", p.Dir) } } } @@ -769,7 +793,7 @@ func installOutsideModule(ctx context.Context, args []string) { pkgOpts := load.PackageOpts{MainOnly: true} pkgs, err := load.PackagesAndErrorsOutsideModule(ctx, pkgOpts, args) if err != nil { - base.Fatalf("go install: %v", err) + base.Fatalf("go: %v", err) } load.CheckPackageErrors(pkgs) patterns := make([]string, len(args)) diff --git a/libgo/go/cmd/go/internal/work/build_test.go b/libgo/go/cmd/go/internal/work/build_test.go index 600fc3083f0..0b6b83a706c 100644 --- a/libgo/go/cmd/go/internal/work/build_test.go +++ b/libgo/go/cmd/go/internal/work/build_test.go @@ -234,7 +234,7 @@ func TestRespectSetgidDir(t *testing.T) { // of `(*Builder).ShowCmd` afterwards as a sanity check. cfg.BuildX = true var cmdBuf bytes.Buffer - b.Print = func(a ...interface{}) (int, error) { + b.Print = func(a ...any) (int, error) { return cmdBuf.WriteString(fmt.Sprint(a...)) } diff --git a/libgo/go/cmd/go/internal/work/buildid.go b/libgo/go/cmd/go/internal/work/buildid.go index 4e9189a3632..76335e9bb17 100644 --- a/libgo/go/cmd/go/internal/work/buildid.go +++ b/libgo/go/cmd/go/internal/work/buildid.go @@ -570,6 +570,8 @@ func showStdout(b *Builder, c *cache.Cache, actionID cache.ActionID, key string) b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID)))) } if !cfg.BuildN { + b.output.Lock() + defer b.output.Unlock() b.Print(string(stdout)) } } @@ -578,6 +580,8 @@ func showStdout(b *Builder, c *cache.Cache, actionID cache.ActionID, key string) // flushOutput flushes the output being queued in a. func (b *Builder) flushOutput(a *Action) { + b.output.Lock() + defer b.output.Unlock() b.Print(string(a.output)) a.output = nil } diff --git a/libgo/go/cmd/go/internal/work/exec.go b/libgo/go/cmd/go/internal/work/exec.go index b0281041c98..d3f0ecac049 100644 --- a/libgo/go/cmd/go/internal/work/exec.go +++ b/libgo/go/cmd/go/internal/work/exec.go @@ -36,6 +36,8 @@ import ( "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/quoted" + "cmd/internal/sys" ) // actionList returns the list of actions in the dag rooted at root @@ -222,18 +224,34 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { // same compiler settings and can reuse each other's results. // If not, the reason is already recorded in buildGcflags. fmt.Fprintf(h, "compile\n") - // Only include the package directory if it may affect the output. - // We trim workspace paths for all packages when -trimpath is set. - // The compiler hides the exact value of $GOROOT - // when building things in GOROOT. - // Assume b.WorkDir is being trimmed properly. - // When -trimpath is used with a package built from the module cache, - // use the module path and version instead of the directory. - if !p.Goroot && !cfg.BuildTrimpath && !strings.HasPrefix(p.Dir, b.WorkDir) { + + // Include information about the origin of the package that + // may be embedded in the debug info for the object file. + if cfg.BuildTrimpath { + // When -trimpath is used with a package built from the module cache, + // its debug information refers to the module path and version + // instead of the directory. + if p.Module != nil { + fmt.Fprintf(h, "module %s@%s\n", p.Module.Path, p.Module.Version) + } + } else if p.Goroot { + // The Go compiler always hides the exact value of $GOROOT + // when building things in GOROOT. + // + // The C compiler does not, but for packages in GOROOT we rewrite the path + // as though -trimpath were set, so that we don't invalidate the build cache + // (and especially any precompiled C archive files) when changing + // GOROOT_FINAL. (See https://go.dev/issue/50183.) + // + // b.WorkDir is always either trimmed or rewritten to + // the literal string "/tmp/go-build". + } else if !strings.HasPrefix(p.Dir, b.WorkDir) { + // -trimpath is not set and no other rewrite rules apply, + // so the object file may refer to the absolute directory + // containing the package. fmt.Fprintf(h, "dir %s\n", p.Dir) - } else if cfg.BuildTrimpath && p.Module != nil { - fmt.Fprintf(h, "module %s@%s\n", p.Module.Path, p.Module.Version) } + if p.Module != nil { fmt.Fprintf(h, "go %s\n", p.Module.GoVersion) } @@ -281,6 +299,11 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { if p.Internal.CoverMode != "" { fmt.Fprintf(h, "cover %q %q\n", p.Internal.CoverMode, b.toolID("cover")) } + if p.Internal.FuzzInstrument { + if fuzzFlags := fuzzInstrumentFlags(); fuzzFlags != nil { + fmt.Fprintf(h, "fuzz %q\n", fuzzFlags) + } + } fmt.Fprintf(h, "modinfo %q\n", p.Internal.BuildInfo) // Configuration specific to compiler toolchain. @@ -775,10 +798,13 @@ OverlayLoop: } if p.Internal.BuildInfo != "" && cfg.ModulesEnabled { - if err := b.writeFile(objdir+"_gomod_.go", modload.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { - return err + prog := modload.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo") + if len(prog) > 0 { + if err := b.writeFile(objdir+"_gomod_.go", prog); err != nil { + return err + } + gofiles = append(gofiles, objdir+"_gomod_.go") } - gofiles = append(gofiles, objdir+"_gomod_.go") } // Compile Go. @@ -1375,6 +1401,7 @@ func (b *Builder) writeLinkImportcfg(a *Action, file string) error { fmt.Fprintf(&icfg, "packageshlib %s=%s\n", p1.ImportPath, p1.Shlib) } } + fmt.Fprintf(&icfg, "modinfo %q\n", modload.ModInfoData(a.Package.Internal.BuildInfo)) return b.writeFile(file, icfg.Bytes()) } @@ -1489,6 +1516,8 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return nil, nil, errPrintedOutput } if len(out) > 0 { + // NOTE: we don't attempt to parse quotes and unescapes here. pkg-config + // is typically used within shell backticks, which treats quotes literally. ldflags = strings.Fields(string(out)) if err := checkLinkerFlags("LDFLAGS", "pkg-config --libs", ldflags); err != nil { return nil, nil, err @@ -1923,7 +1952,7 @@ func mayberemovefile(s string) { // fmtcmd replaces the name of the current directory with dot (.) // but only when it is at the beginning of a space-separated token. // -func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string { +func (b *Builder) fmtcmd(dir string, format string, args ...any) string { cmd := fmt.Sprintf(format, args...) if dir != "" && dir != "/" { dot := " ." @@ -1949,7 +1978,7 @@ func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string // showcmd prints the given command to standard output // for the implementation of -n or -x. -func (b *Builder) Showcmd(dir string, format string, args ...interface{}) { +func (b *Builder) Showcmd(dir string, format string, args ...any) { b.output.Lock() defer b.output.Unlock() b.Print(b.fmtcmd(dir, format, args...) + "\n") @@ -2013,7 +2042,7 @@ var cgoTypeSigRe = lazyregexp.New(`\b_C2?(type|func|var|macro)_\B`) // run runs the command given by cmdline in the directory dir. // If the command fails, run prints information about the failure // and returns a non-nil error. -func (b *Builder) run(a *Action, dir string, desc string, env []string, cmdargs ...interface{}) error { +func (b *Builder) run(a *Action, dir string, desc string, env []string, cmdargs ...any) error { out, err := b.runOut(a, dir, env, cmdargs...) if len(out) > 0 { if desc == "" { @@ -2047,7 +2076,7 @@ func (b *Builder) processOutput(out []byte) string { // runOut runs the command given by cmdline in the directory dir. // It returns the command output and any errors that occurred. // It accumulates execution time in a. -func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...interface{}) ([]byte, error) { +func (b *Builder) runOut(a *Action, dir string, env []string, cmdargs ...any) ([]byte, error) { cmdline := str.StringList(cmdargs...) for _, arg := range cmdline { @@ -2312,7 +2341,7 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s // directives pointing to the source directory. It should not generate those // when -trimpath is enabled. if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { - if cfg.BuildTrimpath { + if cfg.BuildTrimpath || p.Goroot { // Keep in sync with Action.trimpath. // The trimmed paths are a little different, but we need to trim in the // same situations. @@ -2334,8 +2363,6 @@ func (b *Builder) ccompile(a *Action, p *load.Package, outfile string, flags []s to = filepath.Join("/_", toPath) } flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+from+"="+to) - } else if p.Goroot && cfg.GOROOT_FINAL != cfg.GOROOT { - flags = append(flags[:len(flags):len(flags)], "-fdebug-prefix-map="+cfg.GOROOT+"="+cfg.GOROOT_FINAL) } } @@ -2384,7 +2411,7 @@ func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flag cmd = b.GccCmd(p.Dir, objdir) } - cmdargs := []interface{}{cmd, "-o", outfile, objs, flags} + cmdargs := []any{cmd, "-o", outfile, objs, flags} dir := p.Dir out, err := b.runOut(a, base.Cwd(), b.cCompilerEnv(), cmdargs...) @@ -2431,12 +2458,6 @@ func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flag return err } -// Grab these before main helpfully overwrites them. -var ( - origCC = cfg.Getenv("CC") - origCXX = cfg.Getenv("CXX") -) - // gccCmd returns a gcc command line prefix // defaultCC is defined in zdefaultcc.go, written by cmd/dist. func (b *Builder) GccCmd(incdir, workdir string) []string { @@ -2456,40 +2477,23 @@ func (b *Builder) gfortranCmd(incdir, workdir string) []string { // ccExe returns the CC compiler setting without all the extra flags we add implicitly. func (b *Builder) ccExe() []string { - return b.compilerExe(origCC, cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + return envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) } // cxxExe returns the CXX compiler setting without all the extra flags we add implicitly. func (b *Builder) cxxExe() []string { - return b.compilerExe(origCXX, cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + return envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) } // fcExe returns the FC compiler setting without all the extra flags we add implicitly. func (b *Builder) fcExe() []string { - return b.compilerExe(cfg.Getenv("FC"), "gfortran") -} - -// compilerExe returns the compiler to use given an -// environment variable setting (the value not the name) -// and a default. The resulting slice is usually just the name -// of the compiler but can have additional arguments if they -// were present in the environment value. -// For example if CC="gcc -DGOPHER" then the result is ["gcc", "-DGOPHER"]. -func (b *Builder) compilerExe(envValue string, def string) []string { - compiler := strings.Fields(envValue) - if len(compiler) == 0 { - compiler = strings.Fields(def) - } - return compiler + return envList("FC", "gfortran") } // compilerCmd returns a command line prefix for the given environment // variable and using the default command when the variable is empty. func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []string { - // NOTE: env.go's mkEnv knows that the first three - // strings returned are "gcc", "-I", incdir (and cuts them off). - a := []string{compiler[0], "-I", incdir} - a = append(a, compiler[1:]...) + a := append(compiler, "-I", incdir) // Definitely want -fPIC but on Windows gcc complains // "-fPIC ignored for target (all code is position independent)" @@ -2670,12 +2674,20 @@ func (b *Builder) gccArchArgs() []string { // envList returns the value of the given environment variable broken // into fields, using the default value when the variable is empty. +// +// The environment variable must be quoted correctly for +// str.SplitQuotedFields. This should be done before building +// anything, for example, in BuildInit. func envList(key, def string) []string { v := cfg.Getenv(key) if v == "" { v = def } - return strings.Fields(v) + args, err := quoted.Split(v) + if err != nil { + panic(fmt.Sprintf("could not parse environment variable %s with value %q: %v", key, v, err)) + } + return args } // CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. @@ -2741,6 +2753,10 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) } + if cfg.BuildASan { + cgoCFLAGS = append([]string{"-fsanitize=address"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=address"}, cgoLDFLAGS...) + } // Allows including _cgo_export.h, as well as the user's .h files, // from .[ch] files in the package. @@ -2762,7 +2778,7 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = append(cgoflags, "-import_runtime_cgo=false") } - if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo" || p.ImportPath == "runtime/asan") { cgoflags = append(cgoflags, "-import_syscall=false") } @@ -2988,18 +3004,24 @@ func (b *Builder) dynimport(a *Action, p *load.Package, objdir, importGo, cgoExe linkobj := str.StringList(ofile, outObj, mkAbsFiles(p.Dir, p.SysoFiles)) dynobj := objdir + "_cgo_.o" - // we need to use -pie for Linux/ARM to get accurate imported sym ldflags := cgoLDFLAGS if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { - // -static -pie doesn't make sense, and causes link errors. - // Issue 26197. - n := make([]string, 0, len(ldflags)) - for _, flag := range ldflags { - if flag != "-static" { - n = append(n, flag) + if !str.Contains(ldflags, "-no-pie") { + // we need to use -pie for Linux/ARM to get accurate imported sym (added in https://golang.org/cl/5989058) + // this seems to be outdated, but we don't want to break existing builds depending on this (Issue 45940) + ldflags = append(ldflags, "-pie") + } + if str.Contains(ldflags, "-pie") && str.Contains(ldflags, "-static") { + // -static -pie doesn't make sense, and causes link errors. + // Issue 26197. + n := make([]string, 0, len(ldflags)-1) + for _, flag := range ldflags { + if flag != "-static" { + n = append(n, flag) + } } + ldflags = n } - ldflags = append(n, "-pie") } if err := b.gccld(a, p, objdir, dynobj, ldflags, linkobj); err != nil { return err @@ -3321,12 +3343,6 @@ func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { return cleanup } -// Windows has a limit of 32 KB arguments. To be conservative and not worry -// about whether that includes spaces or not, just use 30 KB. Darwin's limit is -// less clear. The OS claims 256KB, but we've seen failures with arglen as -// small as 50KB. -const ArgLengthForResponseFile = (30 << 10) - func useResponseFile(path string, argLen int) bool { // Unless the program uses objabi.Flagparse, which understands // response files, don't use response files. @@ -3338,7 +3354,7 @@ func useResponseFile(path string, argLen int) bool { return false } - if argLen > ArgLengthForResponseFile { + if argLen > sys.ExecArgLengthLimit { return true } diff --git a/libgo/go/cmd/go/internal/work/exec_test.go b/libgo/go/cmd/go/internal/work/exec_test.go index 4eb762cb289..8bbf25bb337 100644 --- a/libgo/go/cmd/go/internal/work/exec_test.go +++ b/libgo/go/cmd/go/internal/work/exec_test.go @@ -7,6 +7,7 @@ package work import ( "bytes" "cmd/internal/objabi" + "cmd/internal/sys" "fmt" "math/rand" "testing" @@ -56,7 +57,7 @@ func TestEncodeDecodeFuzz(t *testing.T) { } t.Parallel() - nRunes := ArgLengthForResponseFile + 100 + nRunes := sys.ExecArgLengthLimit + 100 rBuffer := make([]rune, nRunes) buf := bytes.NewBuffer([]byte(string(rBuffer))) @@ -67,7 +68,7 @@ func TestEncodeDecodeFuzz(t *testing.T) { for i := 0; i < 50; i++ { // Generate a random string of runes. buf.Reset() - for buf.Len() < ArgLengthForResponseFile+1 { + for buf.Len() < sys.ExecArgLengthLimit+1 { var r rune for { r = rune(rng.Intn(utf8.MaxRune + 1)) diff --git a/libgo/go/cmd/go/internal/work/gc.go b/libgo/go/cmd/go/internal/work/gc.go index 85da4f89f99..40175324d26 100644 --- a/libgo/go/cmd/go/internal/work/gc.go +++ b/libgo/go/cmd/go/internal/work/gc.go @@ -22,6 +22,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/objabi" + "cmd/internal/quoted" "cmd/internal/sys" "crypto/sha1" ) @@ -29,6 +30,18 @@ import ( // The 'path' used for GOROOT_FINAL when -trimpath is specified const trimPathGoRootFinal = "go" +var runtimePackages = map[string]struct{}{ + "internal/abi": struct{}{}, + "internal/bytealg": struct{}{}, + "internal/cpu": struct{}{}, + "internal/goarch": struct{}{}, + "internal/goos": struct{}{}, + "runtime": struct{}{}, + "runtime/internal/atomic": struct{}{}, + "runtime/internal/math": struct{}{}, + "runtime/internal/sys": struct{}{}, +} + // The Go toolchain. type gcToolchain struct{} @@ -63,7 +76,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg } pkgpath := pkgPath(a) - gcargs := []string{"-p", pkgpath} + defaultGcFlags := []string{"-p", pkgpath} if p.Module != nil { v := p.Module.GoVersion if v == "" { @@ -82,22 +95,19 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg v = "1.16" } if allowedVersion(v) { - gcargs = append(gcargs, "-lang=go"+v) + defaultGcFlags = append(defaultGcFlags, "-lang=go"+v) } } if p.Standard { - gcargs = append(gcargs, "-std") - } - compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) - // The runtime package imports a couple of general internal packages. - if p.Standard && (p.ImportPath == "internal/cpu" || p.ImportPath == "internal/bytealg" || p.ImportPath == "internal/abi") { - compilingRuntime = true + defaultGcFlags = append(defaultGcFlags, "-std") } + _, compilingRuntime := runtimePackages[p.ImportPath] + compilingRuntime = compilingRuntime && p.Standard if compilingRuntime { // runtime compiles with a special gc flag to check for // memory allocations that are invalid in the runtime package, // and to implement some special compiler pragmas. - gcargs = append(gcargs, "-+") + defaultGcFlags = append(defaultGcFlags, "-+") } // If we're giving the compiler the entire package (no C etc files), tell it that, @@ -116,25 +126,28 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg } } if extFiles == 0 { - gcargs = append(gcargs, "-complete") + defaultGcFlags = append(defaultGcFlags, "-complete") } if cfg.BuildContext.InstallSuffix != "" { - gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) + defaultGcFlags = append(defaultGcFlags, "-installsuffix", cfg.BuildContext.InstallSuffix) } if a.buildID != "" { - gcargs = append(gcargs, "-buildid", a.buildID) + defaultGcFlags = append(defaultGcFlags, "-buildid", a.buildID) } if p.Internal.OmitDebug || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { - gcargs = append(gcargs, "-dwarf=false") + defaultGcFlags = append(defaultGcFlags, "-dwarf=false") } if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { - gcargs = append(gcargs, "-goversion", runtimeVersion) + defaultGcFlags = append(defaultGcFlags, "-goversion", runtimeVersion) } if symabis != "" { - gcargs = append(gcargs, "-symabis", symabis) + defaultGcFlags = append(defaultGcFlags, "-symabis", symabis) } gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags) + if p.Internal.FuzzInstrument { + gcflags = append(gcflags, fuzzInstrumentFlags()...) + } if compilingRuntime { // Remove -N, if present. // It is not possible to build the runtime with no optimizations, @@ -147,10 +160,15 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg } } } + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + gcflags = append(gcflags, fmt.Sprintf("-c=%d", c)) + } - args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), gcflags, gcargs} - if p.Internal.LocalPrefix != "" { - // Workaround #43883. + args := []any{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), defaultGcFlags, gcflags} + if p.Internal.LocalPrefix == "" { + args = append(args, "-nolocalimports") + } else { args = append(args, "-D", p.Internal.LocalPrefix) } if importcfg != nil { @@ -172,11 +190,6 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg args = append(args, "-asmhdr", objdir+"go_asm.h") } - // Add -c=N to use concurrent backend compilation, if possible. - if c := gcBackendConcurrency(gcflags); c > 1 { - args = append(args, fmt.Sprintf("-c=%d", c)) - } - for _, f := range gofiles { f := mkAbs(p.Dir, f) @@ -223,7 +236,7 @@ CheckFlags: // except for known commonly used flags. // If the user knows better, they can manually add their own -c to the gcflags. switch flag { - case "-N", "-l", "-S", "-B", "-C", "-I": + case "-N", "-l", "-S", "-B", "-C", "-I", "-shared": // OK default: canDashC = false @@ -349,11 +362,11 @@ func (a *Action) trimpath() string { return rewrite } -func asmArgs(a *Action, p *load.Package) []interface{} { +func asmArgs(a *Action, p *load.Package) []any { // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. inc := filepath.Join(cfg.GOROOT, "pkg", "include") pkgpath := pkgPath(a) - args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} + args := []any{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} if p.ImportPath == "runtime" && cfg.Goarch == "386" { for _, arg := range forcedAsmflags { if arg == "-dynlink" { @@ -365,6 +378,16 @@ func asmArgs(a *Action, p *load.Package) []interface{} { args = append(args, "-compiling-runtime") } + if cfg.Goarch == "386" { + // Define GO386_value from cfg.GO386. + args = append(args, "-D", "GO386_"+cfg.GO386) + } + + if cfg.Goarch == "amd64" { + // Define GOAMD64_value from cfg.GOAMD64. + args = append(args, "-D", "GOAMD64_"+cfg.GOAMD64) + } + if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { // Define GOMIPS_value from cfg.GOMIPS. args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS) @@ -432,8 +455,8 @@ func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, erro // toolVerify checks that the command line args writes the same output file // if run using newTool instead. // Unused now but kept around for future use. -func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { - newArgs := make([]interface{}, len(args)) +func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []any) error { + newArgs := make([]any, len(args)) copy(newArgs, args) newArgs[1] = base.Tool(newTool) newArgs[3] = ofile + ".new" // x.6 becomes x.6.new @@ -536,33 +559,18 @@ func packInternal(afile string, ofiles []string) error { } // setextld sets the appropriate linker flags for the specified compiler. -func setextld(ldflags []string, compiler []string) []string { +func setextld(ldflags []string, compiler []string) ([]string, error) { for _, f := range ldflags { if f == "-extld" || strings.HasPrefix(f, "-extld=") { // don't override -extld if supplied - return ldflags + return ldflags, nil } } - ldflags = append(ldflags, "-extld="+compiler[0]) - if len(compiler) > 1 { - extldflags := false - add := strings.Join(compiler[1:], " ") - for i, f := range ldflags { - if f == "-extldflags" && i+1 < len(ldflags) { - ldflags[i+1] = add + " " + ldflags[i+1] - extldflags = true - break - } else if strings.HasPrefix(f, "-extldflags=") { - ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] - extldflags = true - break - } - } - if !extldflags { - ldflags = append(ldflags, "-extldflags="+add) - } + joined, err := quoted.Join(compiler) + if err != nil { + return nil, err } - return ldflags + return append(ldflags, "-extld="+joined), nil } // pluginPath computes the package path for a plugin main package. @@ -649,7 +657,10 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) } ldflags = append(ldflags, forcedLdflags...) ldflags = append(ldflags, root.Package.Internal.Ldflags...) - ldflags = setextld(ldflags, compiler) + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } // On OS X when using external linking to build a shared library, // the argument passed here to -o ends up recorded in the final @@ -693,7 +704,10 @@ func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, } else { compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) } - ldflags = setextld(ldflags, compiler) + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } for _, d := range toplevelactions { if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries continue diff --git a/libgo/go/cmd/go/internal/work/init.go b/libgo/go/cmd/go/internal/work/init.go index 37a3e2d0ffd..26192ecaed1 100644 --- a/libgo/go/cmd/go/internal/work/init.go +++ b/libgo/go/cmd/go/internal/work/init.go @@ -11,8 +11,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" "cmd/go/internal/modload" + "cmd/internal/quoted" "cmd/internal/sys" - "flag" "fmt" "os" "path/filepath" @@ -32,27 +32,63 @@ func BuildInit() { if cfg.BuildPkgdir != "" && !filepath.IsAbs(cfg.BuildPkgdir) { p, err := filepath.Abs(cfg.BuildPkgdir) if err != nil { - fmt.Fprintf(os.Stderr, "go %s: evaluating -pkgdir: %v\n", flag.Args()[0], err) + fmt.Fprintf(os.Stderr, "go: evaluating -pkgdir: %v\n", err) base.SetExitStatus(2) base.Exit() } cfg.BuildPkgdir = p } - // Make sure CC and CXX are absolute paths - for _, key := range []string{"CC", "CXX"} { - if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { - base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) + if cfg.BuildP <= 0 { + base.Fatalf("go: -p must be a positive integer: %v\n", cfg.BuildP) + } + + // Make sure CC, CXX, and FC are absolute paths. + for _, key := range []string{"CC", "CXX", "FC"} { + value := cfg.Getenv(key) + args, err := quoted.Split(value) + if err != nil { + base.Fatalf("go: %s environment variable could not be parsed: %v", key, err) } + if len(args) == 0 { + continue + } + path := args[0] + if !filepath.IsAbs(path) && path != filepath.Base(path) { + base.Fatalf("go: %s environment variable is relative; must be absolute path: %s\n", key, path) + } + } +} + +// fuzzInstrumentFlags returns compiler flags that enable fuzzing instrumation +// on supported platforms. +// +// On unsupported platforms, fuzzInstrumentFlags returns nil, meaning no +// instrumentation is added. 'go test -fuzz' still works without coverage, +// but it generates random inputs without guidance, so it's much less effective. +func fuzzInstrumentFlags() []string { + if !sys.FuzzInstrumented(cfg.Goos, cfg.Goarch) { + return nil } + return []string{"-d=libfuzzer"} } func instrumentInit() { - if !cfg.BuildRace && !cfg.BuildMSan { + if !cfg.BuildRace && !cfg.BuildMSan && !cfg.BuildASan { return } if cfg.BuildRace && cfg.BuildMSan { - fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) + fmt.Fprintf(os.Stderr, "go: may not use -race and -msan simultaneously\n") + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildRace && cfg.BuildASan { + fmt.Fprintf(os.Stderr, "go: may not use -race and -asan simultaneously\n") + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildMSan && cfg.BuildASan { + fmt.Fprintf(os.Stderr, "go: may not use -msan and -asan simultaneously\n") base.SetExitStatus(2) base.Exit() } @@ -61,12 +97,15 @@ func instrumentInit() { base.SetExitStatus(2) base.Exit() } - if cfg.BuildRace { - if !sys.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { - fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64, darwin/arm64, and windows/amd64\n", flag.Args()[0]) - base.SetExitStatus(2) - base.Exit() - } + if cfg.BuildRace && !sys.RaceDetectorSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-race is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() + } + if cfg.BuildASan && !sys.ASanSupported(cfg.Goos, cfg.Goarch) { + fmt.Fprintf(os.Stderr, "-asan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + base.SetExitStatus(2) + base.Exit() } mode := "race" if cfg.BuildMSan { @@ -77,13 +116,16 @@ func instrumentInit() { cfg.BuildBuildmode = "pie" } } + if cfg.BuildASan { + mode = "asan" + } modeFlag := "-" + mode if !cfg.BuildContext.CgoEnabled { if runtime.GOOS != cfg.Goos || runtime.GOARCH != cfg.Goarch { - fmt.Fprintf(os.Stderr, "go %s: %s requires cgo\n", flag.Args()[0], modeFlag) + fmt.Fprintf(os.Stderr, "go: %s requires cgo\n", modeFlag) } else { - fmt.Fprintf(os.Stderr, "go %s: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0], modeFlag) + fmt.Fprintf(os.Stderr, "go: %s requires cgo; enable cgo by setting CGO_ENABLED=1\n", modeFlag) } base.SetExitStatus(2) @@ -96,7 +138,7 @@ func instrumentInit() { cfg.BuildContext.InstallSuffix += "_" } cfg.BuildContext.InstallSuffix += mode - cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, mode) + cfg.BuildContext.ToolTags = append(cfg.BuildContext.ToolTags, mode) } func buildModeInit() { diff --git a/libgo/go/cmd/go/internal/work/testgo.go b/libgo/go/cmd/go/internal/work/testgo.go index 8b77871b23f..a09b65a23c3 100644 --- a/libgo/go/cmd/go/internal/work/testgo.go +++ b/libgo/go/cmd/go/internal/work/testgo.go @@ -5,7 +5,6 @@ // This file contains extra hooks for testing the go command. //go:build testgo -// +build testgo package work diff --git a/libgo/go/cmd/go/internal/workcmd/edit.go b/libgo/go/cmd/go/internal/workcmd/edit.go new file mode 100644 index 00000000000..c42000710e0 --- /dev/null +++ b/libgo/go/cmd/go/internal/workcmd/edit.go @@ -0,0 +1,315 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go work edit + +package workcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/modload" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/module" + + "golang.org/x/mod/modfile" +) + +var cmdEdit = &base.Command{ + UsageLine: "go work edit [editing flags] [go.work]", + Short: "edit go.work from tools or scripts", + Long: `Edit provides a command-line interface for editing go.work, +for use primarily by tools or scripts. It only reads go.work; +it does not look up information about the modules involved. +If no file is specified, Edit looks for a go.work file in the current +directory and its parent directories + +The editing flags specify a sequence of editing operations. + +The -fmt flag reformats the go.work file without making other changes. +This reformatting is also implied by any other modifications that use or +rewrite the go.mod file. The only time this flag is needed is if no other +flags are specified, as in 'go work edit -fmt'. + +The -use=path and -dropuse=path flags +add and drop a use directive from the go.work file's set of module directories. + +The -replace=old[@v]=new[@v] flag adds a replacement of the given +module path and version pair. If the @v in old@v is omitted, a +replacement without a version on the left side is added, which applies +to all versions of the old module path. If the @v in new@v is omitted, +the new path should be a local module root directory, not a module +path. Note that -replace overrides any redundant replacements for old[@v], +so omitting @v will drop existing replacements for specific versions. + +The -dropreplace=old[@v] flag drops a replacement of the given +module path and version pair. If the @v is omitted, a replacement without +a version on the left side is dropped. + +The -use, -dropuse, -replace, and -dropreplace, +editing flags may be repeated, and the changes are applied in the order given. + +The -go=version flag sets the expected Go language version. + +The -print flag prints the final go.work in its text format instead of +writing it back to go.mod. + +The -json flag prints the final go.work file in JSON format instead of +writing it back to go.mod. The JSON output corresponds to these Go types: + + type Module struct { + Path string + Version string + } + + type GoWork struct { + Go string + Directory []Directory + Replace []Replace + } + + type Use struct { + Path string + ModulePath string + } + + type Replace struct { + Old Module + New Module + } + +See the workspaces design proposal at +https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for +more information. +`, +} + +var ( + editFmt = cmdEdit.Flag.Bool("fmt", false, "") + editGo = cmdEdit.Flag.String("go", "", "") + editJSON = cmdEdit.Flag.Bool("json", false, "") + editPrint = cmdEdit.Flag.Bool("print", false, "") + workedits []func(file *modfile.WorkFile) // edits specified in flags +) + +type flagFunc func(string) + +func (f flagFunc) String() string { return "" } +func (f flagFunc) Set(s string) error { f(s); return nil } + +func init() { + cmdEdit.Run = runEditwork // break init cycle + + cmdEdit.Flag.Var(flagFunc(flagEditworkUse), "use", "") + cmdEdit.Flag.Var(flagFunc(flagEditworkDropUse), "dropuse", "") + cmdEdit.Flag.Var(flagFunc(flagEditworkReplace), "replace", "") + cmdEdit.Flag.Var(flagFunc(flagEditworkDropReplace), "dropreplace", "") + + base.AddWorkfileFlag(&cmdEdit.Flag) +} + +func runEditwork(ctx context.Context, cmd *base.Command, args []string) { + anyFlags := + *editGo != "" || + *editJSON || + *editPrint || + *editFmt || + len(workedits) > 0 + + if !anyFlags { + base.Fatalf("go: no flags specified (see 'go help work edit').") + } + + if *editJSON && *editPrint { + base.Fatalf("go: cannot use both -json and -print") + } + + if len(args) > 1 { + base.Fatalf("go: 'go help work edit' accepts at most one argument") + } + var gowork string + if len(args) == 1 { + gowork = args[0] + } else { + modload.InitWorkfile() + gowork = modload.WorkFilePath() + } + + if *editGo != "" { + if !modfile.GoVersionRE.MatchString(*editGo) { + base.Fatalf(`go mod: invalid -go option; expecting something like "-go %s"`, modload.LatestGoVersion()) + } + } + + workFile, err := modload.ReadWorkFile(gowork) + if err != nil { + base.Fatalf("go: errors parsing %s:\n%s", base.ShortPath(gowork), err) + } + + if *editGo != "" { + if err := workFile.AddGoStmt(*editGo); err != nil { + base.Fatalf("go: internal error: %v", err) + } + } + + if len(workedits) > 0 { + for _, edit := range workedits { + edit(workFile) + } + } + + modload.UpdateWorkFile(workFile) + + workFile.SortBlocks() + workFile.Cleanup() // clean file after edits + + if *editJSON { + editPrintJSON(workFile) + return + } + + if *editPrint { + os.Stdout.Write(modfile.Format(workFile.Syntax)) + return + } + + modload.WriteWorkFile(gowork, workFile) +} + +// flagEditworkUse implements the -use flag. +func flagEditworkUse(arg string) { + workedits = append(workedits, func(f *modfile.WorkFile) { + _, mf, err := modload.ReadModFile(filepath.Join(arg, "go.mod"), nil) + modulePath := "" + if err == nil { + modulePath = mf.Module.Mod.Path + } + f.AddUse(modload.ToDirectoryPath(arg), modulePath) + if err := f.AddUse(modload.ToDirectoryPath(arg), ""); err != nil { + base.Fatalf("go: -use=%s: %v", arg, err) + } + }) +} + +// flagEditworkDropUse implements the -dropuse flag. +func flagEditworkDropUse(arg string) { + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.DropUse(modload.ToDirectoryPath(arg)); err != nil { + base.Fatalf("go: -dropdirectory=%s: %v", arg, err) + } + }) +} + +// allowedVersionArg returns whether a token may be used as a version in go.mod. +// We don't call modfile.CheckPathVersion, because that insists on versions +// being in semver form, but here we want to allow versions like "master" or +// "1234abcdef", which the go command will resolve the next time it runs (or +// during -fix). Even so, we need to make sure the version is a valid token. +func allowedVersionArg(arg string) bool { + return !modfile.MustQuote(arg) +} + +// parsePathVersionOptional parses path[@version], using adj to +// describe any errors. +func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version string, err error) { + if i := strings.Index(arg, "@"); i < 0 { + path = arg + } else { + path, version = strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:]) + } + if err := module.CheckImportPath(path); err != nil { + if !allowDirPath || !modfile.IsDirectoryPath(path) { + return path, version, fmt.Errorf("invalid %s path: %v", adj, err) + } + } + if path != arg && !allowedVersionArg(version) { + return path, version, fmt.Errorf("invalid %s version: %q", adj, version) + } + return path, version, nil +} + +// flagReplace implements the -replace flag. +func flagEditworkReplace(arg string) { + var i int + if i = strings.Index(arg, "="); i < 0 { + base.Fatalf("go: -replace=%s: need old[@v]=new[@w] (missing =)", arg) + } + old, new := strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:]) + if strings.HasPrefix(new, ">") { + base.Fatalf("go: -replace=%s: separator between old and new is =, not =>", arg) + } + oldPath, oldVersion, err := parsePathVersionOptional("old", old, false) + if err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + newPath, newVersion, err := parsePathVersionOptional("new", new, true) + if err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + if newPath == new && !modfile.IsDirectoryPath(new) { + base.Fatalf("go: -replace=%s: unversioned new path must be local directory", arg) + } + + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.AddReplace(oldPath, oldVersion, newPath, newVersion); err != nil { + base.Fatalf("go: -replace=%s: %v", arg, err) + } + }) +} + +// flagDropReplace implements the -dropreplace flag. +func flagEditworkDropReplace(arg string) { + path, version, err := parsePathVersionOptional("old", arg, true) + if err != nil { + base.Fatalf("go: -dropreplace=%s: %v", arg, err) + } + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.DropReplace(path, version); err != nil { + base.Fatalf("go: -dropreplace=%s: %v", arg, err) + } + }) +} + +type replaceJSON struct { + Old module.Version + New module.Version +} + +// editPrintJSON prints the -json output. +func editPrintJSON(workFile *modfile.WorkFile) { + var f workfileJSON + if workFile.Go != nil { + f.Go = workFile.Go.Version + } + for _, d := range workFile.Use { + f.Use = append(f.Use, useJSON{DiskPath: d.Path, ModPath: d.ModulePath}) + } + + for _, r := range workFile.Replace { + f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) + } + data, err := json.MarshalIndent(&f, "", "\t") + if err != nil { + base.Fatalf("go: internal error: %v", err) + } + data = append(data, '\n') + os.Stdout.Write(data) +} + +// workfileJSON is the -json output data structure. +type workfileJSON struct { + Go string `json:",omitempty"` + Use []useJSON + Replace []replaceJSON +} + +type useJSON struct { + DiskPath string + ModPath string `json:",omitempty"` +} diff --git a/libgo/go/cmd/go/internal/workcmd/init.go b/libgo/go/cmd/go/internal/workcmd/init.go new file mode 100644 index 00000000000..cefecee8329 --- /dev/null +++ b/libgo/go/cmd/go/internal/workcmd/init.go @@ -0,0 +1,52 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go work init + +package workcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/modload" + "context" + "path/filepath" +) + +var cmdInit = &base.Command{ + UsageLine: "go work init [moddirs]", + Short: "initialize workspace file", + Long: `Init initializes and writes a new go.work file in the +current directory, in effect creating a new workspace at the current +directory. + +go work init optionally accepts paths to the workspace modules as +arguments. If the argument is omitted, an empty workspace with no +modules will be created. + +Each argument path is added to a use directive in the go.work file. The +current go version will also be listed in the go.work file. + +`, + Run: runInit, +} + +func init() { + base.AddModCommonFlags(&cmdInit.Flag) + base.AddWorkfileFlag(&cmdInit.Flag) +} + +func runInit(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + modload.ForceUseModules = true + + // TODO(matloob): support using the -workfile path + // To do that properly, we'll have to make the module directories + // make dirs relative to workFile path before adding the paths to + // the directory entries + + workFile := filepath.Join(base.Cwd(), "go.work") + + modload.CreateWorkFile(ctx, workFile, args) +} diff --git a/libgo/go/cmd/go/internal/workcmd/sync.go b/libgo/go/cmd/go/internal/workcmd/sync.go new file mode 100644 index 00000000000..1cca817517c --- /dev/null +++ b/libgo/go/cmd/go/internal/workcmd/sync.go @@ -0,0 +1,130 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go work sync + +package workcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/imports" + "cmd/go/internal/modload" + "context" + + "golang.org/x/mod/module" +) + +var cmdSync = &base.Command{ + UsageLine: "go work sync", + Short: "sync workspace build list to modules", + Long: `Sync syncs the workspace's build list back to the +workspace's modules + +The workspace's build list is the set of versions of all the +(transitive) dependency modules used to do builds in the workspace. go +work sync generates that build list using the Minimal Version Selection +algorithm, and then syncs those versions back to each of modules +specified in the workspace (with use directives). + +The syncing is done by sequentially upgrading each of the dependency +modules specified in a workspace module to the version in the build list +if the dependency module's version is not already the same as the build +list's version. Note that Minimal Version Selection guarantees that the +build list's version of each module is always the same or higher than +that in each workspace module. +`, + Run: runSync, +} + +func init() { + base.AddModCommonFlags(&cmdSync.Flag) + base.AddWorkfileFlag(&cmdSync.Flag) +} + +func runSync(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + modload.ForceUseModules = true + + workGraph := modload.LoadModGraph(ctx, "") + _ = workGraph + mustSelectFor := map[module.Version][]module.Version{} + + mms := modload.MainModules + + opts := modload.PackageOpts{ + Tags: imports.AnyTags(), + VendorModulesInGOROOTSrc: true, + ResolveMissingImports: false, + LoadTests: true, + AllowErrors: true, + SilencePackageErrors: true, + SilenceUnmatchedWarnings: true, + } + for _, m := range mms.Versions() { + opts.MainModule = m + _, pkgs := modload.LoadPackages(ctx, opts, "all") + opts.MainModule = module.Version{} // reset + + var ( + mustSelect []module.Version + inMustSelect = map[module.Version]bool{} + ) + for _, pkg := range pkgs { + if r := modload.PackageModule(pkg); r.Version != "" && !inMustSelect[r] { + // r has a known version, so force that version. + mustSelect = append(mustSelect, r) + inMustSelect[r] = true + } + } + module.Sort(mustSelect) // ensure determinism + mustSelectFor[m] = mustSelect + } + + workFilePath := modload.WorkFilePath() // save go.work path because EnterModule clobbers it. + + for _, m := range mms.Versions() { + if mms.ModRoot(m) == "" && m.Path == "command-line-arguments" { + // This is not a real module. + // TODO(#49228): Remove this special case once the special + // command-line-arguments module is gone. + continue + } + + // Use EnterModule to reset the global state in modload to be in + // single-module mode using the modroot of m. + modload.EnterModule(ctx, mms.ModRoot(m)) + + // Edit the build list in the same way that 'go get' would if we + // requested the relevant module versions explicitly. + changed, err := modload.EditBuildList(ctx, nil, mustSelectFor[m]) + if err != nil { + base.Errorf("go: %v", err) + } + if !changed { + continue + } + + modload.LoadPackages(ctx, modload.PackageOpts{ + Tags: imports.AnyTags(), + Tidy: true, + VendorModulesInGOROOTSrc: true, + ResolveMissingImports: false, + LoadTests: true, + AllowErrors: true, + SilenceMissingStdImports: true, + SilencePackageErrors: true, + }, "all") + modload.WriteGoMod(ctx) + } + + wf, err := modload.ReadWorkFile(workFilePath) + if err != nil { + base.Fatalf("go: %v", err) + } + modload.UpdateWorkFile(wf) + if err := modload.WriteWorkFile(workFilePath, wf); err != nil { + base.Fatalf("go: %v", err) + } +} diff --git a/libgo/go/cmd/go/internal/workcmd/use.go b/libgo/go/cmd/go/internal/workcmd/use.go new file mode 100644 index 00000000000..852e5b910c2 --- /dev/null +++ b/libgo/go/cmd/go/internal/workcmd/use.go @@ -0,0 +1,119 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go work use + +package workcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/fsys" + "cmd/go/internal/modload" + "context" + "io/fs" + "os" + "path/filepath" +) + +var cmdUse = &base.Command{ + UsageLine: "go work use [-r] [moddirs]", + Short: "add modules to workspace file", + Long: `Use provides a command-line interface for adding +directories, optionally recursively, to a go.work file. + +A use directive will be added to the go.work file for each argument +directory listed on the command line go.work file, if it exists on disk, +or removed from the go.work file if it does not exist on disk. + +The -r flag searches recursively for modules in the argument +directories, and the use command operates as if each of the directories +were specified as arguments: namely, use directives will be added for +directories that exist, and removed for directories that do not exist. +`, +} + +var useR = cmdUse.Flag.Bool("r", false, "") + +func init() { + cmdUse.Run = runUse // break init cycle + + base.AddModCommonFlags(&cmdUse.Flag) + base.AddWorkfileFlag(&cmdUse.Flag) +} + +func runUse(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + modload.ForceUseModules = true + + var gowork string + modload.InitWorkfile() + gowork = modload.WorkFilePath() + + workFile, err := modload.ReadWorkFile(gowork) + if err != nil { + base.Fatalf("go: %v", err) + } + + haveDirs := make(map[string]bool) + for _, dir := range workFile.Use { + haveDirs[filepath.Join(filepath.Dir(gowork), filepath.FromSlash(dir.Path))] = true + } + + addDirs := make(map[string]bool) + removeDirs := make(map[string]bool) + lookDir := func(dir string) { + absDir := filepath.Join(base.Cwd(), dir) + // If the path is absolute, keep it absolute. If it's relative, + // make it relative to the go.work file rather than the working directory. + if !filepath.IsAbs(dir) { + rel, err := filepath.Rel(filepath.Dir(gowork), absDir) + if err == nil { + dir = rel + } + } + fi, err := os.Stat(filepath.Join(dir, "go.mod")) + if err != nil { + if os.IsNotExist(err) { + + if haveDirs[absDir] { + removeDirs[dir] = true + } + return + } + base.Errorf("go: %v", err) + } + + if !fi.Mode().IsRegular() { + base.Errorf("go: %v is not regular", filepath.Join(dir, "go.mod")) + } + + if !haveDirs[absDir] { + addDirs[dir] = true + } + } + + for _, useDir := range args { + if *useR { + fsys.Walk(useDir, func(path string, info fs.FileInfo, err error) error { + if !info.IsDir() { + return nil + } + lookDir(path) + return nil + }) + continue + } + lookDir(useDir) + } + + for dir := range removeDirs { + workFile.DropUse(filepath.ToSlash(dir)) + } + for dir := range addDirs { + workFile.AddUse(filepath.ToSlash(dir), "") + } + modload.UpdateWorkFile(workFile) + modload.WriteWorkFile(gowork, workFile) +} diff --git a/libgo/go/cmd/go/internal/workcmd/work.go b/libgo/go/cmd/go/internal/workcmd/work.go new file mode 100644 index 00000000000..5bb0a2e8bad --- /dev/null +++ b/libgo/go/cmd/go/internal/workcmd/work.go @@ -0,0 +1,72 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package workcmd implements the ``go work'' command. +package workcmd + +import ( + "cmd/go/internal/base" +) + +var CmdWork = &base.Command{ + UsageLine: "go work", + Short: "workspace maintenance", + Long: `Go workspace provides access to operations on workspaces. + +Note that support for workspaces is built into many other commands, not +just 'go work'. + +See 'go help modules' for information about Go's module system of which +workspaces are a part. + +A workspace is specified by a go.work file that specifies a set of +module directories with the "use" directive. These modules are used as +root modules by the go command for builds and related operations. A +workspace that does not specify modules to be used cannot be used to do +builds from local modules. + +go.work files are line-oriented. Each line holds a single directive, +made up of a keyword followed by aruments. For example: + + go 1.18 + + use ../foo/bar + use ./baz + + replace example.com/foo v1.2.3 => example.com/bar v1.4.5 + +The leading keyword can be factored out of adjacent lines to create a block, +like in Go imports. + + use ( + ../foo/bar + ./baz + ) + +The use directive specifies a module to be included in the workspace's +set of main modules. The argument to the use directive is the directory +containing the module's go.mod file. + +The go directive specifies the version of Go the file was written at. It +is possible there may be future changes in the semantics of workspaces +that could be controlled by this version, but for now the version +specified has no effect. + +The replace directive has the same syntax as the replace directive in a +go.mod file and takes precedence over replaces in go.mod files. It is +primarily intended to override conflicting replaces in different workspace +modules. + +To determine whether the go command is operating in workspace mode, use +the "go env GOWORK" command. This will specify the workspace file being +used. +`, + + Commands: []*base.Command{ + cmdEdit, + cmdInit, + cmdSync, + cmdUse, + }, +} |