diff options
Diffstat (limited to 'src/cmd/go/internal')
46 files changed, 1310 insertions, 908 deletions
diff --git a/src/cmd/go/internal/base/base.go b/src/cmd/go/internal/base/base.go index db3ebef933..004588c732 100644 --- a/src/cmd/go/internal/base/base.go +++ b/src/cmd/go/internal/base/base.go @@ -56,6 +56,20 @@ var Go = &Command{ // Commands initialized in package main } +// hasFlag reports whether a command or any of its subcommands contain the given +// flag. +func hasFlag(c *Command, name string) bool { + if f := c.Flag.Lookup(name); f != nil { + return true + } + for _, sub := range c.Commands { + if hasFlag(sub, name) { + return true + } + } + return false +} + // LongName returns the command's long name: all the words in the usage line between "go" and a flag or argument, func (c *Command) LongName() string { name := c.UsageLine diff --git a/src/cmd/go/internal/base/goflags.go b/src/cmd/go/internal/base/goflags.go index 34766134b0..4da27550fd 100644 --- a/src/cmd/go/internal/base/goflags.go +++ b/src/cmd/go/internal/base/goflags.go @@ -13,15 +13,7 @@ import ( "cmd/go/internal/cfg" ) -var ( - goflags []string // cached $GOFLAGS list; can be -x or --x form - knownFlag = make(map[string]bool) // flags allowed to appear in $GOFLAGS; no leading dashes -) - -// AddKnownFlag adds name to the list of known flags for use in $GOFLAGS. -func AddKnownFlag(name string) { - knownFlag[name] = true -} +var goflags []string // cached $GOFLAGS list; can be -x or --x form // GOFLAGS returns the flags from $GOFLAGS. // The list can be assumed to contain one string per flag, @@ -38,22 +30,12 @@ func InitGOFLAGS() { return } - // Build list of all flags for all commands. - // If no command has that flag, then we report the problem. - // This catches typos while still letting users record flags in GOFLAGS - // that only apply to a subset of go commands. - // Commands using CustomFlags can report their flag names - // by calling AddKnownFlag instead. - var walkFlags func(*Command) - walkFlags = func(cmd *Command) { - for _, sub := range cmd.Commands { - walkFlags(sub) - } - cmd.Flag.VisitAll(func(f *flag.Flag) { - knownFlag[f.Name] = true - }) + goflags = strings.Fields(cfg.Getenv("GOFLAGS")) + if len(goflags) == 0 { + // nothing to do; avoid work on later InitGOFLAGS call + goflags = []string{} + return } - walkFlags(Go) // Ignore bad flag in go env and go bug, because // they are what people reach for when debugging @@ -61,11 +43,6 @@ func InitGOFLAGS() { // (Both will show the GOFLAGS setting if let succeed.) hideErrors := cfg.CmdName == "env" || cfg.CmdName == "bug" - goflags = strings.Fields(cfg.Getenv("GOFLAGS")) - if goflags == nil { - goflags = []string{} // avoid work on later InitGOFLAGS call - } - // Each of the words returned by strings.Fields must be its own flag. // To set flag arguments use -x=value instead of -x value. // For boolean flags, -x is fine instead of -x=true. @@ -85,7 +62,7 @@ func InitGOFLAGS() { if i := strings.Index(name, "="); i >= 0 { name = name[:i] } - if !knownFlag[name] { + if !hasFlag(Go, name) { if hideErrors { continue } @@ -153,3 +130,20 @@ func SetFromGOFLAGS(flags *flag.FlagSet) { } } } + +// InGOFLAGS returns whether GOFLAGS contains the given flag, such as "-mod". +func InGOFLAGS(flag string) bool { + for _, goflag := range GOFLAGS() { + name := goflag + if strings.HasPrefix(name, "--") { + name = name[1:] + } + if i := strings.Index(name, "="); i >= 0 { + name = name[:i] + } + if name == flag { + return true + } + } + return false +} diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go index 52bd40f2fb..07e3516855 100644 --- a/src/cmd/go/internal/bug/bug.go +++ b/src/cmd/go/internal/bug/bug.go @@ -105,7 +105,7 @@ func printGoDetails(w io.Writer) { func printOSDetails(w io.Writer) { switch runtime.GOOS { - case "darwin": + case "darwin", "ios": printCmdOut(w, "uname -v: ", "uname", "-v") printCmdOut(w, "", "sw_vers") case "linux": diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index f874b880a6..9bf1db73ef 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -49,6 +49,8 @@ var ( ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag + Insecure bool // -insecure flag + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index d0be3fe1e7..ed2786879c 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -18,6 +18,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/vcs" "cmd/go/internal/web" "cmd/go/internal/work" @@ -43,10 +44,11 @@ The -fix flag instructs get to run the fix tool on the downloaded packages before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. The -GOINSECURE environment variable is usually a better alternative, since it -provides control over which modules may be retrieved using an insecure scheme. -See 'go help environment' for details. +custom domains using insecure schemes such as HTTP. Use with caution. +This flag is deprecated and will be removed in a future version of go. +The GOINSECURE environment variable is usually a better alternative, since +it provides control over which modules may be retrieved using an insecure +scheme. See 'go help environment' for details. The -t flag instructs get to also download the packages required to build the tests for the specified packages. @@ -108,14 +110,12 @@ var ( getT = CmdGet.Flag.Bool("t", false, "") getU = CmdGet.Flag.Bool("u", false, "") getFix = CmdGet.Flag.Bool("fix", false, "") - - Insecure bool ) func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") } func runGet(ctx context.Context, cmd *base.Command, args []string) { @@ -129,6 +129,9 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if *getF && !*getU { base.Fatalf("go get: cannot use -f flag without -u") } + if cfg.Insecure { + fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n") + } // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding @@ -408,7 +411,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // to make the first copy of or update a copy of the given package. func downloadPackage(p *load.Package) error { var ( - vcs *vcsCmd + vcsCmd *vcs.Cmd repo, rootPath string err error blindRepo bool // set if the repo has unusual configuration @@ -431,22 +434,22 @@ func downloadPackage(p *load.Package) error { return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err) } security := web.SecureOnly - if Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) { + if cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) { security = web.Insecure } if p.Internal.Build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. - vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot) + vcsCmd, rootPath, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot) if err != nil { return err } repo = "<local>" // should be unused; make distinctive // Double-check where it came from. - if *getU && vcs.remoteRepo != nil { + if *getU && vcsCmd.RemoteRepo != nil { dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - remote, err := vcs.remoteRepo(vcs, dir) + remote, err := vcsCmd.RemoteRepo(vcsCmd, dir) if err != nil { // Proceed anyway. The package is present; we likely just don't understand // the repo configuration (e.g. unusual remote protocol). @@ -454,10 +457,10 @@ func downloadPackage(p *load.Package) error { } repo = remote if !*getF && err == nil { - if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil { + if rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security); err == nil { repo := rr.Repo - if rr.vcs.resolveRepo != nil { - resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) + if rr.VCS.ResolveRepo != nil { + resolved, err := rr.VCS.ResolveRepo(rr.VCS, dir, repo) if err == nil { repo = resolved } @@ -471,13 +474,13 @@ func downloadPackage(p *load.Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security) if err != nil { return err } - vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root + vcsCmd, repo, rootPath = rr.VCS, rr.Repo, rr.Root } - if !blindRepo && !vcs.isSecure(repo) && security != web.Insecure { + if !blindRepo && !vcsCmd.IsSecure(repo) && security != web.Insecure { return fmt.Errorf("cannot download, %v uses insecure protocol", repo) } @@ -500,7 +503,7 @@ func downloadPackage(p *load.Package) error { } root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil { + if err := vcs.CheckNested(vcsCmd, root, p.Internal.Build.SrcRoot); err != nil { return err } @@ -516,7 +519,7 @@ func downloadPackage(p *load.Package) error { // Check that this is an appropriate place for the repo to be checked out. // The target directory must either not exist or have a repo checked out already. - meta := filepath.Join(root, "."+vcs.cmd) + meta := filepath.Join(root, "."+vcsCmd.Cmd) if _, err := os.Stat(meta); err != nil { // Metadata file or directory does not exist. Prepare to checkout new copy. // Some version control tools require the target directory not to exist. @@ -537,12 +540,12 @@ func downloadPackage(p *load.Package) error { fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root) } - if err = vcs.create(root, repo); err != nil { + if err = vcsCmd.Create(root, repo); err != nil { return err } } else { // Metadata directory does exist; download incremental updates. - if err = vcs.download(root); err != nil { + if err = vcsCmd.Download(root); err != nil { return err } } @@ -551,12 +554,12 @@ func downloadPackage(p *load.Package) error { // Do not show tag sync in -n; it's noise more than anything, // and since we're not running commands, no tag will be found. // But avoid printing nothing. - fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) + fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcsCmd.Cmd) return nil } // Select and sync to appropriate version of the repository. - tags, err := vcs.tags(root) + tags, err := vcsCmd.Tags(root) if err != nil { return err } @@ -564,7 +567,7 @@ func downloadPackage(p *load.Package) error { if i := strings.Index(vers, " "); i >= 0 { vers = vers[:i] } - if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { + if err := vcsCmd.TagSync(root, selectTag(vers, tags)); err != nil { return err } diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index e1f0521ea4..0ae5fd7ca9 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -838,6 +838,9 @@ in addition to android tags and files. Using GOOS=illumos matches build tags and files as for GOOS=solaris in addition to illumos tags and files. +Using GOOS=ios matches build tags and files as for GOOS=darwin +in addition to ios tags and files. + To keep a file from being considered for the build: // +build ignore diff --git a/src/cmd/go/internal/imports/build.go b/src/cmd/go/internal/imports/build.go index eb070eef4c..50aeabc578 100644 --- a/src/cmd/go/internal/imports/build.go +++ b/src/cmd/go/internal/imports/build.go @@ -141,6 +141,9 @@ func matchTag(name string, tags map[string]bool, want bool) bool { if name == "solaris" { have = have || tags["illumos"] } + if name == "darwin" { + have = have || tags["ios"] + } return have == want } @@ -158,6 +161,7 @@ func matchTag(name string, tags map[string]bool, want bool) bool { // Exceptions: // if GOOS=android, then files with GOOS=linux are also matched. // if GOOS=illumos, then files with GOOS=solaris are also matched. +// if GOOS=ios, then files with GOOS=darwin are also matched. // // If tags["*"] is true, then MatchFile will consider all possible // GOOS and GOARCH to be available and will consequently @@ -208,6 +212,7 @@ var KnownOS = map[string]bool{ "freebsd": true, "hurd": true, "illumos": true, + "ios": true, "js": true, "linux": true, "nacl": true, // legacy; don't remove diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 23500dd9d8..33409eb774 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -325,7 +325,7 @@ var ( var nl = []byte{'\n'} func runList(ctx context.Context, cmd *base.Command, args []string) { - modload.LoadTests = *listTest + load.ModResolveTests = *listTest work.BuildInit() out := newTrackingWriter(os.Stdout) defer out.w.Flush() diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 71fd9b5538..5cc77915e7 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -28,27 +28,13 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" + "cmd/go/internal/modload" "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" ) -var ( - // module initialization hook; never nil, no-op if module use is disabled - ModInit func() - - // module hooks; nil if module use is disabled - ModBinDir func() string // return effective bin directory - ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import - ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths - ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary - ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files - ModDirImportPath func(string) string // return effective import path for directory -) - var IgnoreImports bool // control whether we ignore imports in packages // A Package describes a single package found in a directory. @@ -434,13 +420,16 @@ type ImportPathError interface { ImportPath() string } +var ( + _ ImportPathError = (*importError)(nil) + _ ImportPathError = (*modload.ImportMissingError)(nil) +) + type importError struct { importPath string err error // created with fmt.Errorf } -var _ ImportPathError = (*importError)(nil) - func ImportErrorf(path, format string, args ...interface{}) ImportPathError { err := &importError{importPath: path, err: fmt.Errorf(format, args...)} if errStr := err.Error(); !strings.Contains(errStr, path) { @@ -753,7 +742,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd // For vendored imports, it is the expanded form. // // Note that when modules are enabled, local import paths are normally - // canonicalized by modload.ImportPaths before now. However, if there's an + // canonicalized by modload.LoadPackages before now. However, if there's an // error resolving a local path, it will be returned untransformed // so that 'go list -e' reports something useful. importKey := importSpec{ @@ -770,7 +759,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd r.dir = filepath.Join(parentDir, path) r.path = dirToImportPath(r.dir) } else if cfg.ModulesEnabled { - r.dir, r.path, r.err = ModLookup(parentPath, parentIsStd, path) + r.dir, r.path, r.err = modload.Lookup(parentPath, parentIsStd, path) } else if mode&ResolveImport != 0 { // We do our own path resolution, because we want to // find out the key to use in packageCache without the @@ -801,7 +790,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd } data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode) if data.p.Root == "" && cfg.ModulesEnabled { - if info := ModPackageModuleInfo(path); info != nil { + if info := modload.PackageModuleInfo(path); info != nil { data.p.Root = info.Dir } } @@ -827,7 +816,7 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd if cfg.GOBIN != "" { data.p.BinDir = cfg.GOBIN } else if cfg.ModulesEnabled { - data.p.BinDir = ModBinDir() + data.p.BinDir = modload.BinDir() } } @@ -895,8 +884,8 @@ var preloadWorkerCount = runtime.GOMAXPROCS(0) // to ensure preload goroutines are no longer active. This is necessary // because of global mutable state that cannot safely be read and written // concurrently. In particular, packageDataCache may be cleared by "go get" -// in GOPATH mode, and modload.loaded (accessed via ModLookup) may be -// modified by modload.ImportPaths (ModImportPaths). +// in GOPATH mode, and modload.loaded (accessed via modload.Lookup) may be +// modified by modload.LoadPackages. type preload struct { cancel chan struct{} sema chan struct{} @@ -1006,7 +995,7 @@ func ResolveImportPath(parent *Package, path string) (found string) { func resolveImportPath(path, parentPath, parentDir, parentRoot string, parentIsStd bool) (found string) { if cfg.ModulesEnabled { - if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil { + if _, p, e := modload.Lookup(parentPath, parentIsStd, path); e == nil { return p } return path @@ -1369,7 +1358,7 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * // directory containing them. // If the directory is outside the main module, this will resolve to ".", // which is not a prefix of any valid module. - importerPath = ModDirImportPath(importer.Dir) + importerPath = modload.DirImportPath(importer.Dir) } parentOfInternal := p.ImportPath[:i] if str.HasPathPrefix(importerPath, parentOfInternal) { @@ -1652,7 +1641,7 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor elem = full } if p.Internal.Build.BinDir == "" && cfg.ModulesEnabled { - p.Internal.Build.BinDir = ModBinDir() + p.Internal.Build.BinDir = modload.BinDir() } if p.Internal.Build.BinDir != "" { // Install to GOBIN or bin of GOPATH entry. @@ -1861,9 +1850,9 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor if p.Internal.CmdlineFiles { mainPath = "command-line-arguments" } - p.Module = ModPackageModuleInfo(mainPath) + p.Module = modload.PackageModuleInfo(mainPath) if p.Name == "main" && len(p.DepsErrors) == 0 { - p.Internal.BuildInfo = ModPackageBuildInfo(mainPath, p.Deps) + p.Internal.BuildInfo = modload.PackageBuildInfo(mainPath, p.Deps) } } } @@ -1961,7 +1950,7 @@ func externalLinkingForced(p *Package) bool { if cfg.BuildContext.GOARCH != "arm64" { return true } - case "darwin": + case "darwin", "ios": if cfg.BuildContext.GOARCH == "arm64" { return true } @@ -2117,6 +2106,18 @@ func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, return p } +// ModResolveTests indicates whether calls to the module loader should also +// resolve test dependencies of the requested packages. +// +// If ModResolveTests is true, then the module loader needs to resolve test +// dependencies at the same time as packages; otherwise, the test dependencies +// of those packages could be missing, and resolving those missing dependencies +// could change the selected versions of modules that provide other packages. +// +// TODO(#40775): Change this from a global variable to an explicit function +// argument where needed. +var ModResolveTests bool + // Packages returns the packages named by the // command line arguments 'args'. If a named package // cannot be loaded at all (for example, if the directory does not exist), @@ -2158,7 +2159,18 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { } } - matches := ImportPaths(ctx, patterns) + var matches []*search.Match + if modload.Init(); cfg.ModulesEnabled { + loadOpts := modload.PackageOpts{ + ResolveMissingImports: true, + LoadTests: ModResolveTests, + SilenceErrors: true, + } + matches, _ = modload.LoadPackages(ctx, loadOpts, patterns...) + } else { + matches = search.ImportPaths(patterns) + } + var ( pkgs []*Package stk ImportStack @@ -2228,13 +2240,6 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(ctx context.Context, args []string) []*search.Match { - if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(ctx, args) - } - return search.ImportPaths(args) -} - // PackagesForBuild is like Packages but exits // if any of the packages or their dependencies have errors // (cannot be built). @@ -2282,7 +2287,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. func GoFilesPackage(ctx context.Context, gofiles []string) *Package { - ModInit() + modload.Init() for _, f := range gofiles { if !strings.HasSuffix(f, ".go") { @@ -2329,7 +2334,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(ctx, gofiles) + modload.ImportFromFiles(ctx, gofiles) } var err error @@ -2357,7 +2362,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { if cfg.GOBIN != "" { pkg.Target = filepath.Join(cfg.GOBIN, exe) } else if cfg.ModulesEnabled { - pkg.Target = filepath.Join(ModBinDir(), exe) + pkg.Target = filepath.Join(modload.BinDir(), exe) } } diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go index c60a78ed92..8776c5741c 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix solaris +// +build aix solaris,!illumos // This code implements the filelock API using POSIX 'fcntl' locks, which attach // to an (inode, process) pair rather than a file descriptor. To avoid unlocking @@ -12,9 +12,6 @@ // Most platforms provide some alternative API, such as an 'flock' system call // or an F_OFD_SETLK command for 'fcntl', that allows for better concurrency and // does not require per-inode bookkeeping in the application. -// -// TODO(golang.org/issue/35618): add a syscall.Flock binding for Illumos and -// switch it over to use filelock_unix.go. package filelock diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go index faf73446f7..8301fb6b6e 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go @@ -161,7 +161,7 @@ func TestRLockExcludesOnlyLock(t *testing.T) { doUnlockTF := false switch runtime.GOOS { - case "aix", "illumos", "solaris": + case "aix", "solaris": // When using POSIX locks (as on Solaris), we can't safely read-lock the // same inode through two different descriptors at the same time: when the // first descriptor is closed, the second descriptor would still be open but diff --git a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go index 00c4262832..78f2c51129 100644 --- a/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go +++ b/src/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd +// +build darwin dragonfly freebsd illumos linux netbsd openbsd package filelock diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 6227fd9f33..050a2e0e12 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -80,9 +80,7 @@ type moduleJSON struct { func runDownload(ctx context.Context, cmd *base.Command, args []string) { // Check whether modules are enabled and whether we're in a module. - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } + modload.ForceUseModules = true if !modload.HasModRoot() && len(args) == 0 { base.Fatalf("go mod download: no modules specified (see 'go help mod download')") } diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index a149b65605..3277548c23 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -13,7 +13,6 @@ import ( "sort" "cmd/go/internal/base" - "cmd/go/internal/cfg" "cmd/go/internal/modload" "golang.org/x/mod/module" @@ -39,14 +38,8 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod graph: graph takes no arguments") } - // Checks go mod expected behavior - if !modload.Enabled() { - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } else { - base.Fatalf("go: cannot find main module; see 'go help modules'") - } - } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot modload.LoadAllModules(ctx) reqs := modload.MinReqs() diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 21b235653e..7cfc0e6f5b 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -40,9 +40,7 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 1 { modload.CmdModModule = args[0] } - if os.Getenv("GO111MODULE") == "off" { - base.Fatalf("go mod init: modules disabled by GO111MODULE=off; see 'go help modules'") - } + modload.ForceUseModules = true modFilePath := modload.ModFilePath() if _, err := os.Stat(modFilePath); err == nil { base.Fatalf("go mod init: go.mod already exists") diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index 30df674ef6..fb43e33ec5 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -9,12 +9,13 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/imports" "cmd/go/internal/modload" "context" ) var cmdTidy = &base.Command{ - UsageLine: "go mod tidy [-v]", + UsageLine: "go mod tidy [-e] [-v]", Short: "add missing and remove unused modules", Long: ` Tidy makes sure go.mod matches the source code in the module. @@ -25,12 +26,18 @@ to go.sum and removes any unnecessary ones. The -v flag causes tidy to print information about removed modules to standard error. + +The -e flag causes tidy to attempt to proceed despite errors +encountered while loading packages. `, + Run: runTidy, } +var tidyE bool // if true, report errors but proceed anyway. + func init() { - cmdTidy.Run = runTidy // break init cycle cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "") + cmdTidy.Flag.BoolVar(&tidyE, "e", false, "") base.AddModCommonFlags(&cmdTidy.Flag) } @@ -49,9 +56,16 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { // those packages. In order to make 'go test' reproducible for the packages // that are in 'all' but outside of the main module, we must explicitly // request that their test dependencies be included. - modload.LoadTests = true + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + modload.LoadPackages(ctx, modload.PackageOpts{ + Tags: imports.AnyTags(), + ResolveMissingImports: true, + LoadTests: true, + AllowErrors: tidyE, + }, "all") - modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 91d2509452..1bc4ab3def 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -25,7 +25,7 @@ import ( ) var cmdVendor = &base.Command{ - UsageLine: "go mod vendor [-v]", + UsageLine: "go mod vendor [-e] [-v]", Short: "make vendored copy of dependencies", Long: ` Vendor resets the main module's vendor directory to include all packages @@ -34,12 +34,18 @@ It does not include test code for vendored packages. The -v flag causes vendor to print the names of vendored modules and packages to standard error. + +The -e flag causes vendor to attempt to proceed despite errors +encountered while loading packages. `, Run: runVendor, } +var vendorE bool // if true, report errors but proceed anyway + func init() { cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") + cmdVendor.Flag.BoolVar(&vendorE, "e", false, "") base.AddModCommonFlags(&cmdVendor.Flag) } @@ -47,7 +53,16 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor(ctx) + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + ResolveMissingImports: true, + UseVendorAll: true, + AllowErrors: vendorE, + } + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 7700588bde..bd591d3f32 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -14,7 +14,6 @@ import ( "runtime" "cmd/go/internal/base" - "cmd/go/internal/cfg" "cmd/go/internal/modfetch" "cmd/go/internal/modload" @@ -45,14 +44,8 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) { // NOTE(rsc): Could take a module pattern. base.Fatalf("go mod verify: verify takes no arguments") } - // Checks go mod expected behavior - if !modload.Enabled() || !modload.HasModRoot() { - if cfg.Getenv("GO111MODULE") == "off" { - base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") - } else { - base.Fatalf("go: cannot find main module; see 'go help modules'") - } - } + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot // Only verify up to GOMAXPROCS zips at once. type token struct{} diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 8454fdfec6..e287c88060 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -10,6 +10,7 @@ import ( "strings" "cmd/go/internal/base" + "cmd/go/internal/imports" "cmd/go/internal/modload" "golang.org/x/mod/module" @@ -61,12 +62,16 @@ func init() { } func runWhy(ctx context.Context, cmd *base.Command, args []string) { - loadALL := modload.LoadALL - if *whyVendor { - loadALL = modload.LoadVendor - } else { - modload.LoadTests = true + modload.ForceUseModules = true + modload.RootMode = modload.NeedRoot + + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + LoadTests: !*whyVendor, + SilenceErrors: true, + UseVendorAll: *whyVendor, } + if *whyM { listU := false listVersions := false @@ -78,7 +83,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) byModule := make(map[module.Version][]string) - for _, path := range loadALL(ctx) { + _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") + for _, path := range pkgs { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -107,8 +113,11 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(ctx, args) // resolve to packages - loadALL(ctx) // rebuild graph, from main module (not from named packages) + // Resolve to packages. + matches, _ := modload.LoadPackages(ctx, loadOpts, args...) + + modload.LoadPackages(ctx, loadOpts, "all") // rebuild graph, from main module (not from named packages) + sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modfetch/insecure.go b/src/cmd/go/internal/modfetch/insecure.go index b692669cba..012d05f29d 100644 --- a/src/cmd/go/internal/modfetch/insecure.go +++ b/src/cmd/go/internal/modfetch/insecure.go @@ -6,12 +6,11 @@ package modfetch import ( "cmd/go/internal/cfg" - "cmd/go/internal/get" "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) + return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 34f805d58a..eed4dd4258 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -13,9 +13,9 @@ import ( "time" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" + "cmd/go/internal/vcs" web "cmd/go/internal/web" "golang.org/x/mod/module" @@ -261,13 +261,13 @@ func lookupDirect(path string) (Repo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. return nil, notExistError{err: err} } - if rr.VCS == "mod" { + if rr.VCS.Name == "mod" { // Fetch module from proxy with base URL rr.Repo. return newProxyRepo(rr.Repo, path) } @@ -279,8 +279,8 @@ func lookupDirect(path string) (Repo, error) { return newCodeRepo(code, rr.Root, path) } -func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { - code, err := codehost.NewRepo(rr.VCS, rr.Repo) +func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { + code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) if err != nil { if _, ok := err.(*codehost.VCSError); ok { return nil, err @@ -306,7 +306,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security) if err != nil { return nil, nil, err } diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 783c4a433b..47a2571531 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -22,7 +22,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/lockedfile" "cmd/go/internal/web" @@ -33,7 +32,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index a2a8287d84..f1cf8b17a8 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -17,7 +17,7 @@ import ( "sync" "cmd/go/internal/base" - "cmd/go/internal/get" + "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/load" "cmd/go/internal/modload" @@ -115,9 +115,12 @@ require downgrading other dependencies, and 'go get' does this automatically as well. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. The -GOINSECURE environment variable is usually a better alternative, since it -provides control over which modules may be retrieved using an insecure scheme. +custom domains using insecure schemes such as HTTP. Use with caution. +This flag is deprecated and will be removed in a future version of go. +The GOINSECURE environment variable is usually a better alternative, since +it provides control over which modules may be retrieved using an insecure +scheme. It should be noted that the -insecure flag also turns the module +checksum validation off. GOINSECURE does not do that, use GONOSUMDB. See 'go help environment' for details. The second step is to download (if needed), build, and install @@ -181,7 +184,7 @@ var ( getM = CmdGet.Flag.Bool("m", false, "") getT = CmdGet.Flag.Bool("t", false, "") getU upgradeFlag - // -insecure is get.Insecure + // -insecure is cfg.Insecure // -v is cfg.BuildV ) @@ -206,7 +209,7 @@ func (v *upgradeFlag) String() string { return "" } func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&get.Insecure, "insecure", get.Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") CmdGet.Flag.Var(&getU, "u", "") } @@ -225,6 +228,8 @@ type getArg struct { vers string } +func (arg getArg) String() string { return arg.raw } + // querySpec describes a query for a specific module. path may be a // module path, package path, or package pattern. vers is a version // query string from a command line argument. @@ -276,14 +281,10 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if *getM { base.Fatalf("go get: -m flag is no longer supported; consider -d to skip building packages") } - modload.LoadTests = *getT - - buildList := modload.LoadAllModules(ctx) - buildList = buildList[:len(buildList):len(buildList)] // copy on append - versionByPath := make(map[string]string) - for _, m := range buildList { - versionByPath[m.Path] = m.Version + if cfg.Insecure { + fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n") } + load.ModResolveTests = *getT // Do not allow any updating of go.mod until we've applied // all the requested changes and checked that the result matches @@ -294,150 +295,15 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // 'go get' is expected to do this, unlike other commands. modload.AllowMissingModuleImports() - // Parse command-line arguments and report errors. The command-line - // arguments are of the form path@version or simply path, with implicit - // @upgrade. path@none is "downgrade away". - var gets []getArg - var queries []*query - for _, arg := range search.CleanPatterns(args) { - // Argument is path or path@vers. - path := arg - vers := "" - if i := strings.Index(arg, "@"); i >= 0 { - path, vers = arg[:i], arg[i+1:] - } - if strings.Contains(vers, "@") || arg != path && vers == "" { - base.Errorf("go get %s: invalid module version syntax", arg) - continue - } - - // Guard against 'go get x.go', a common mistake. - // Note that package and module paths may end with '.go', so only print an error - // if the argument has no version and either has no slash or refers to an existing file. - if strings.HasSuffix(arg, ".go") && vers == "" { - if !strings.Contains(arg, "/") { - base.Errorf("go get %s: arguments must be package or module paths", arg) - continue - } - if fi, err := os.Stat(arg); err == nil && !fi.IsDir() { - base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", arg) - continue - } - } - - // If no version suffix is specified, assume @upgrade. - // If -u=patch was specified, assume @patch instead. - if vers == "" { - if getU != "" { - vers = string(getU) - } else { - vers = "upgrade" - } - } + getArgs := parseArgs(args) - gets = append(gets, getArg{raw: arg, path: path, vers: vers}) - - // Determine the modules that path refers to, and create queries - // to lookup modules at target versions before loading packages. - // This is an imprecise process, but it helps reduce unnecessary - // queries and package loading. It's also necessary for handling - // patterns like golang.org/x/tools/..., which can't be expanded - // during package loading until they're in the build list. - switch { - case filepath.IsAbs(path) || search.IsRelativePath(path): - // Absolute paths like C:\foo and relative paths like ../foo... - // are restricted to matching packages in the main module. If the path - // is explicit and contains no wildcards (...), check that it is a - // package in the main module. If the path contains wildcards but - // matches no packages, we'll warn after package loading. - if !strings.Contains(path, "...") { - m := search.NewMatch(path) - if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(ctx, pkgPath) - } - if len(m.Pkgs) == 0 { - for _, err := range m.Errs { - base.Errorf("go get %s: %v", arg, err) - } - - abs, err := filepath.Abs(path) - if err != nil { - abs = path - } - base.Errorf("go get %s: path %s is not a package in module rooted at %s", arg, abs, modload.ModRoot()) - continue - } - } - - if path != arg { - base.Errorf("go get %s: can't request explicit version of path in main module", arg) - continue - } - - case strings.Contains(path, "..."): - // Wait until we load packages to look up modules. - // We don't know yet whether any modules in the build list provide - // packages matching the pattern. For example, suppose - // golang.org/x/tools and golang.org/x/tools/playground are separate - // modules, and only golang.org/x/tools is in the build list. If the - // user runs 'go get golang.org/x/tools/playground/...', we should - // add a requirement for golang.org/x/tools/playground. We should not - // upgrade golang.org/x/tools. - - case path == "all": - // If there is no main module, "all" is not meaningful. - if !modload.HasModRoot() { - base.Errorf(`go get %s: cannot match "all": working directory is not part of a module`, arg) - } - // Don't query modules until we load packages. We'll automatically - // look up any missing modules. - - case search.IsMetaPackage(path): - base.Errorf("go get %s: explicit requirement on standard-library module %s not allowed", path, path) - continue - - default: - // The argument is a package or module path. - if modload.HasModRoot() { - if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { - // The path is in the main module. Nothing to query. - if vers != "upgrade" && vers != "patch" { - base.Errorf("go get %s: can't request explicit version of path in main module", arg) - } - continue - } - } - - first := path - if i := strings.IndexByte(first, '/'); i >= 0 { - first = path - } - if !strings.Contains(first, ".") { - // The path doesn't have a dot in the first component and cannot be - // queried as a module. It may be a package in the standard library, - // which is fine, so don't report an error unless we encounter - // a problem loading packages below. - continue - } - - // If we're querying "upgrade" or "patch", we need to know the current - // version of the module. For "upgrade", we want to avoid accidentally - // downgrading from a newer prerelease. For "patch", we need to query - // the correct minor version. - // Here, we check if "path" is the name of a module in the build list - // (other than the main module) and set prevM if so. If "path" isn't - // a module in the build list, the current version doesn't matter - // since it's either an unknown module or a package within a module - // that we'll discover later. - q := &query{querySpec: querySpec{path: path, vers: vers}, arg: arg} - if v, ok := versionByPath[path]; ok && path != modload.Target.Path { - q.prevM = module.Version{Path: path, Version: v} - q.forceModulePath = true - } - queries = append(queries, q) - } + buildList := modload.LoadAllModules(ctx) + buildList = buildList[:len(buildList):len(buildList)] // copy on append + selectedVersion := make(map[string]string) + for _, m := range buildList { + selectedVersion[m.Path] = m.Version } - base.ExitIfErrors() + queries := classifyArgs(ctx, selectedVersion, getArgs) // Query modules referenced by command line arguments at requested versions. // We need to do this before loading packages since patterns that refer to @@ -448,13 +314,13 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Add missing modules to the build list. // We call SetBuildList here and elsewhere, since newUpgrader, - // ImportPathsQuiet, and other functions read the global build list. + // LoadPackages, and other functions read the global build list. for _, q := range queries { - if _, ok := versionByPath[q.m.Path]; !ok && q.m.Version != "none" { + if _, ok := selectedVersion[q.m.Path]; !ok && q.m.Version != "none" { buildList = append(buildList, q.m) } } - versionByPath = nil // out of date now; rebuilt later when needed + selectedVersion = nil // out of date now; rebuilt later when needed modload.SetBuildList(buildList) // Upgrade modules specifically named on the command line. This is our only @@ -508,7 +374,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Build a list of arguments that may refer to packages. var pkgPatterns []string var pkgGets []getArg - for _, arg := range gets { + for _, arg := range getArgs { if modOnly[arg.path] == nil && arg.vers != "none" { pkgPatterns = append(pkgPatterns, arg.path) pkgGets = append(pkgGets, arg) @@ -534,9 +400,16 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if len(pkgPatterns) > 0 { // Don't load packages if pkgPatterns is empty. Both - // modload.ImportPathsQuiet and ModulePackages convert an empty list + // modload.LoadPackages and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + ResolveMissingImports: true, // dubious; see https://golang.org/issue/32567 + LoadTests: *getT, + SilenceErrors: true, // Errors may be fixed by subsequent upgrades or downgrades. + SilenceUnmatchedWarnings: true, // We will warn after iterating below. + } + matches, _ = modload.LoadPackages(ctx, loadOpts, pkgPatterns...) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -609,11 +482,6 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } prevBuildList = buildList } - if !*getD { - // Only print warnings after the last iteration, - // and only if we aren't going to build. - search.WarnUnmatched(matches) - } // Handle downgrades. var down []module.Version @@ -640,12 +508,12 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Scan for any upgrades lost by the downgrades. var lostUpgrades []*query if len(down) > 0 { - versionByPath = make(map[string]string) + selectedVersion = make(map[string]string) for _, m := range modload.LoadedModules() { - versionByPath[m.Path] = m.Version + selectedVersion[m.Path] = m.Version } for _, q := range byPath { - if v, ok := versionByPath[q.m.Path]; q.m.Version != "none" && (!ok || semver.Compare(v, q.m.Version) != 0) { + if v, ok := selectedVersion[q.m.Path]; q.m.Version != "none" && (!ok || semver.Compare(v, q.m.Version) != 0) { lostUpgrades = append(lostUpgrades, q) } } @@ -692,7 +560,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if sep != "," { // We have no idea why this happened. // At least report the problem. - if v := versionByPath[q.m.Path]; v == "" { + if v := selectedVersion[q.m.Path]; v == "" { fmt.Fprintf(&buf, " removed unexpectedly") } else { fmt.Fprintf(&buf, " ended up at %s unexpectedly", v) @@ -703,6 +571,37 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("%v", buf.String()) } + if len(pkgPatterns) > 0 || len(args) == 0 { + // Before we write the updated go.mod file, reload the requested packages to + // check for errors. + loadOpts := modload.PackageOpts{ + Tags: imports.AnyTags(), + LoadTests: *getT, + + // Only print warnings after the last iteration, and only if we aren't going + // to build (to avoid doubled warnings). + // + // Only local patterns in the main module, such as './...', can be unmatched. + // (See the mod_get_nopkgs test for more detail.) + SilenceUnmatchedWarnings: !*getD, + } + modload.LoadPackages(ctx, loadOpts, pkgPatterns...) + } + + // If -d was specified, we're done after the module work. + // We've already downloaded modules by loading packages above. + // Otherwise, we need to build and install the packages matched by + // command line arguments. This may be a different set of packages, + // since we only build packages for the target platform. + // Note that 'go get -u' without arguments is equivalent to + // 'go get -u .', so we'll typically build the package in the current + // directory. + if !*getD && len(pkgPatterns) > 0 { + work.BuildInit() + pkgs := load.PackagesForBuild(ctx, pkgPatterns) + work.InstallPackages(ctx, pkgPatterns, pkgs) + } + // Everything succeeded. Update go.mod. modload.AllowWriteGoMod() modload.WriteGoMod() @@ -715,21 +614,174 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // contains information about direct dependencies that WriteGoMod uses. // Refactor to avoid these kinds of global side effects. reportRetractions(ctx) +} - // If -d was specified, we're done after the module work. - // We've already downloaded modules by loading packages above. - // Otherwise, we need to build and install the packages matched by - // command line arguments. This may be a different set of packages, - // since we only build packages for the target platform. - // Note that 'go get -u' without arguments is equivalent to - // 'go get -u .', so we'll typically build the package in the current - // directory. - if *getD || len(pkgPatterns) == 0 { - return +// parseArgs parses command-line arguments and reports errors. +// +// The command-line arguments are of the form path@version or simply path, with +// implicit @upgrade. path@none is "downgrade away". +func parseArgs(rawArgs []string) []getArg { + defer base.ExitIfErrors() + + var gets []getArg + for _, raw := range search.CleanPatterns(rawArgs) { + // Argument is path or path@vers. + path := raw + vers := "" + if i := strings.Index(raw, "@"); i >= 0 { + path, vers = raw[:i], raw[i+1:] + } + if strings.Contains(vers, "@") || raw != path && vers == "" { + base.Errorf("go get %s: invalid module version syntax", raw) + continue + } + + // Guard against 'go get x.go', a common mistake. + // Note that package and module paths may end with '.go', so only print an error + // if the argument has no version and either has no slash or refers to an existing file. + if strings.HasSuffix(raw, ".go") && vers == "" { + if !strings.Contains(raw, "/") { + base.Errorf("go get %s: arguments must be package or module paths", raw) + continue + } + if fi, err := os.Stat(raw); err == nil && !fi.IsDir() { + base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", raw) + continue + } + } + + // If no version suffix is specified, assume @upgrade. + // If -u=patch was specified, assume @patch instead. + if vers == "" { + if getU != "" { + vers = string(getU) + } else { + vers = "upgrade" + } + } + + gets = append(gets, getArg{raw: raw, path: path, vers: vers}) } - work.BuildInit() - pkgs := load.PackagesForBuild(ctx, pkgPatterns) - work.InstallPackages(ctx, pkgPatterns, pkgs) + + return gets +} + +// classifyArgs determines which arguments refer to packages and which refer to +// modules, and creates queries to look up modules at target versions before +// loading packages. +// +// This is an imprecise process, but it helps reduce unnecessary +// queries and package loading. It's also necessary for handling +// patterns like golang.org/x/tools/..., which can't be expanded +// during package loading until they're in the build list. +func classifyArgs(ctx context.Context, selectedVersion map[string]string, args []getArg) []*query { + defer base.ExitIfErrors() + + queries := make([]*query, 0, len(args)) + + for _, arg := range args { + path := arg.path + switch { + case filepath.IsAbs(path) || search.IsRelativePath(path): + // Absolute paths like C:\foo and relative paths like ../foo... + // are restricted to matching packages in the main module. If the path + // is explicit and contains no wildcards (...), check that it is a + // package in the main module. If the path contains wildcards but + // matches no packages, we'll warn after package loading. + if !strings.Contains(path, "...") { + m := search.NewMatch(path) + if pkgPath := modload.DirImportPath(path); pkgPath != "." { + m = modload.TargetPackages(ctx, pkgPath) + } + if len(m.Pkgs) == 0 { + for _, err := range m.Errs { + base.Errorf("go get %s: %v", arg, err) + } + + abs, err := filepath.Abs(path) + if err != nil { + abs = path + } + base.Errorf("go get %s: path %s is not a package in module rooted at %s", arg, abs, modload.ModRoot()) + continue + } + } + + if arg.path != arg.raw { + base.Errorf("go get %s: can't request explicit version of path in main module", arg) + continue + } + + case strings.Contains(path, "..."): + // Wait until we load packages to look up modules. + // We don't know yet whether any modules in the build list provide + // packages matching the pattern. For example, suppose + // golang.org/x/tools and golang.org/x/tools/playground are separate + // modules, and only golang.org/x/tools is in the build list. If the + // user runs 'go get golang.org/x/tools/playground/...', we should + // add a requirement for golang.org/x/tools/playground. We should not + // upgrade golang.org/x/tools. + + case path == "all": + // If there is no main module, "all" is not meaningful. + if !modload.HasModRoot() { + base.Errorf(`go get %s: cannot match "all": working directory is not part of a module`, arg) + } + // Don't query modules until we load packages. We'll automatically + // look up any missing modules. + + case search.IsMetaPackage(path): + base.Errorf("go get %s: explicit requirement on standard-library module %s not allowed", path, path) + continue + + default: + // The argument is a package or module path. + if modload.HasModRoot() { + if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { + // The path is in the main module. Nothing to query. + if arg.vers != "upgrade" && arg.vers != "patch" { + base.Errorf("go get %s: can't request explicit version of path in main module", arg) + } + continue + } + } + + first := path + if i := strings.IndexByte(first, '/'); i >= 0 { + first = path + } + if !strings.Contains(first, ".") { + // The path doesn't have a dot in the first component and cannot be + // queried as a module. It may be a package in the standard library, + // which is fine, so don't report an error unless we encounter + // a problem loading packages. + continue + } + + // If we're querying "upgrade" or "patch", we need to know the current + // version of the module. For "upgrade", we want to avoid accidentally + // downgrading from a newer prerelease. For "patch", we need to query + // the correct minor version. + // Here, we check if "path" is the name of a module in the build list + // (other than the main module) and set prevM if so. If "path" isn't + // a module in the build list, the current version doesn't matter + // since it's either an unknown module or a package within a module + // that we'll discover later. + q := &query{querySpec: querySpec{path: arg.path, vers: arg.vers}, arg: arg.raw} + if v, ok := selectedVersion[path]; ok { + if path == modload.Target.Path { + // TODO(bcmills): This is held over from a previous version of the get + // implementation. Why was it a special case? + } else { + q.prevM = module.Version{Path: path, Version: v} + q.forceModulePath = true + } + } + queries = append(queries, q) + } + } + + return queries } // runQueries looks up modules at target versions in parallel. Results will be diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 9ca6230500..f49b52df56 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -50,7 +50,7 @@ func findStandardImportPath(path string) string { // PackageModuleInfo returns information about the module that provides // a given package. If modules are not enabled or if the package is in the // standard library or if the package was not successfully loaded with -// ImportPaths or a similar loading function, nil is returned. +// LoadPackages or ImportFromFiles, nil is returned. func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if isStandardImportPath(pkgpath) || !Enabled() { return nil @@ -250,8 +250,7 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetrac // PackageBuildInfo returns a string containing module version information // for modules providing packages named by path and deps. path and deps must -// name packages that were resolved successfully with ImportPaths or one of -// the Load functions. +// name packages that were resolved successfully with LoadPackages. func PackageBuildInfo(path string, deps []string) string { if isStandardImportPath(path) || !Enabled() { return "" @@ -321,9 +320,8 @@ func mustFindModule(target, path string) module.Version { } // findModule searches for the module that contains the package at path. -// If the package was loaded with ImportPaths or one of the other loading -// functions, its containing module and true are returned. Otherwise, -// module.Version{} and false are returend. +// If the package was loaded, its containing module and true are returned. +// Otherwise, module.Version{} and false are returend. func findModule(path string) (module.Version, bool) { if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} @@ -355,13 +353,13 @@ func ModInfoProg(info string, isgccgo bool) []byte { import _ "unsafe" //go:linkname __debug_modinfo__ runtime.modinfo var __debug_modinfo__ = %q - `, string(infoStart)+info+string(infoEnd))) +`, string(infoStart)+info+string(infoEnd))) } else { return []byte(fmt.Sprintf(`package main import _ "unsafe" //go:linkname __set_debug_modinfo__ runtime.setmodinfo func __set_debug_modinfo__(string) func init() { __set_debug_modinfo__(%q) } - `, string(infoStart)+info+string(infoEnd))) +`, string(infoStart)+info+string(infoEnd))) } } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 581a1b944a..059b020420 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -17,8 +17,8 @@ import ( ) // buildList is the list of modules to use for building packages. -// It is initialized by calling ImportPaths, ImportFromFiles, -// LoadALL, or LoadBuildList, each of which uses loaded.load. +// It is initialized by calling LoadPackages or ImportFromFiles, +// each of which uses loaded.load. // // Ideally, exactly ONE of those functions would be called, // and exactly once. Most of the time, that's true. @@ -31,8 +31,8 @@ var buildList []module.Version // module pattern, starting with the Target module and in a deterministic // (stable) order, without loading any packages. // -// Modules are loaded automatically (and lazily) in ImportPaths: -// LoadAllModules need only be called if ImportPaths is not, +// Modules are loaded automatically (and lazily) in LoadPackages: +// LoadAllModules need only be called if LoadPackages is not, // typically in commands that care about modules but no particular package. // // The caller must not modify the returned list. @@ -44,7 +44,7 @@ func LoadAllModules(ctx context.Context) []module.Version { } // LoadedModules returns the list of module requirements loaded or set by a -// previous call (typically LoadAllModules or ImportPaths), starting with the +// previous call (typically LoadAllModules or LoadPackages), starting with the // Target module and in a deterministic (stable) order. // // The caller must not modify the returned list. @@ -71,8 +71,8 @@ func ReloadBuildList() []module.Version { } // TidyBuildList trims the build list to the minimal requirements needed to -// retain the same versions of all packages from the preceding Load* or -// ImportPaths* call. +// retain the same versions of all packages from the preceding call to +// LoadPackages. func TidyBuildList() { used := map[module.Version]bool{Target: true} for _, pkg := range loaded.pkgs { diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index 37f23d967f..56920c28b9 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -124,72 +124,63 @@ and the build list. For example: Maintaining module requirements -The go.mod file is meant to be readable and editable by both -programmers and tools. The go command itself automatically updates the go.mod file -to maintain a standard formatting and the accuracy of require statements. - -Any go command that finds an unfamiliar import will look up the module -containing that import and add the latest version of that module -to go.mod automatically. In most cases, therefore, it suffices to -add an import to source code and run 'go build', 'go test', or even 'go list': -as part of analyzing the package, the go command will discover -and resolve the import and update the go.mod file. - -Any go command can determine that a module requirement is -missing and must be added, even when considering only a single -package from the module. On the other hand, determining that a module requirement -is no longer necessary and can be deleted requires a full view of -all packages in the module, across all possible build configurations -(architectures, operating systems, build tags, and so on). -The 'go mod tidy' command builds that view and then -adds any missing module requirements and removes unnecessary ones. +The go.mod file is meant to be readable and editable by both programmers and +tools. Most updates to dependencies can be performed using "go get" and +"go mod tidy". Other module-aware build commands may be invoked using the +-mod=mod flag to automatically add missing requirements and fix inconsistencies. + +The "go get" command updates go.mod to change the module versions used in a +build. An upgrade of one module may imply upgrading others, and similarly a +downgrade of one module may imply downgrading others. The "go get" command +makes these implied changes as well. See "go help module-get". + +The "go mod" command provides other functionality for use in maintaining +and understanding modules and go.mod files. See "go help mod", particularly +"go help mod tidy" and "go help mod edit". As part of maintaining the require statements in go.mod, the go command tracks which ones provide packages imported directly by the current module and which ones provide packages only used indirectly by other module dependencies. Requirements needed only for indirect uses are marked with a -"// indirect" comment in the go.mod file. Indirect requirements are +"// indirect" comment in the go.mod file. Indirect requirements may be automatically removed from the go.mod file once they are implied by other direct requirements. Indirect requirements only arise when using modules that fail to state some of their own dependencies or when explicitly upgrading a module's dependencies ahead of its own stated requirements. -Because of this automatic maintenance, the information in go.mod is an -up-to-date, readable description of the build. - -The 'go get' command updates go.mod to change the module versions used in a -build. An upgrade of one module may imply upgrading others, and similarly a -downgrade of one module may imply downgrading others. The 'go get' command -makes these implied changes as well. If go.mod is edited directly, commands -like 'go build' or 'go list' will assume that an upgrade is intended and -automatically make any implied upgrades and update go.mod to reflect them. - -The 'go mod' command provides other functionality for use in maintaining -and understanding modules and go.mod files. See 'go help mod'. - -The -mod build flag provides additional control over updating and use of go.mod. - -If invoked with -mod=readonly, the go command is disallowed from the implicit -automatic updating of go.mod described above. Instead, it fails when any changes -to go.mod are needed. This setting is most useful to check that go.mod does -not need updates, such as in a continuous integration and testing system. -The "go get" command remains permitted to update go.mod even with -mod=readonly, -and the "go mod" commands do not take the -mod flag (or any other build flags). +The -mod build flag provides additional control over the updating and use of +go.mod for commands that build packages like "go build" and "go test". + +If invoked with -mod=readonly (the default in most situations), the go command +reports an error if a package named on the command line or an imported package +is not provided by any module in the build list computed from the main module's +requirements. The go command also reports an error if a module's checksum is +missing from go.sum (see Module downloading and verification). Either go.mod or +go.sum must be updated in these situations. + +If invoked with -mod=mod, the go command automatically updates go.mod and +go.sum, fixing inconsistencies and adding missing requirements and checksums +as needed. If the go command finds an unfamiliar import, it looks up the +module containing that import and adds a requirement for the latest version +of that module to go.mod. In most cases, therefore, one may add an import to +source code and run "go build", "go test", or even "go list" with -mod=mod: +as part of analyzing the package, the go command will resolve the import and +update the go.mod file. If invoked with -mod=vendor, the go command loads packages from the main module's vendor directory instead of downloading modules to and loading packages from the module cache. The go command assumes the vendor directory holds correct copies of dependencies, and it does not compute the set of required module versions from go.mod files. However, the go command does check that -vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +vendor/modules.txt (generated by "go mod vendor") contains metadata consistent with go.mod. -If invoked with -mod=mod, the go command loads modules from the module cache -even if there is a vendor directory present. +If the go command is not invoked with a -mod flag, and the vendor directory +is present, and the "go" version in go.mod is 1.14 or higher, the go command +will act as if it were invoked with -mod=vendor. Otherwise, the -mod flag +defaults to -mod=readonly. -If the go command is not invoked with a -mod flag and the vendor directory -is present and the "go" version in go.mod is 1.14 or higher, the go command -will act as if it were invoked with -mod=vendor. +Note that neither "go get" nor the "go mod" subcommands accept the -mod flag. Pseudo-versions diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 10b1e7f4b8..c36c8bd29b 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -17,7 +17,6 @@ import ( "time" "cmd/go/internal/cfg" - "cmd/go/internal/load" "cmd/go/internal/modfetch" "cmd/go/internal/par" "cmd/go/internal/search" @@ -38,8 +37,6 @@ type ImportMissingError struct { newMissingVersion string } -var _ load.ImportPathError = (*ImportMissingError)(nil) - func (e *ImportMissingError) Error() string { if e.Module.Path == "" { if search.IsStandardImportPath(e.Path) { @@ -105,8 +102,6 @@ func (e *AmbiguousImportError) Error() string { return buf.String() } -var _ load.ImportPathError = &AmbiguousImportError{} - type invalidImportError struct { importPath string err error @@ -124,8 +119,6 @@ func (e *invalidImportError) Unwrap() error { return e.err } -var _ load.ImportPathError = &invalidImportError{} - // importFromBuildList finds the module and directory in the build list // containing the package with the given import path. The answer must be unique: // importFromBuildList returns an error if multiple modules attempt to provide @@ -268,7 +261,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { } // Every module path in mods is a prefix of the import path. - // As in QueryPackage, prefer the longest prefix that satisfies the import. + // As in QueryPattern, prefer the longest prefix that satisfies the import. sort.Slice(mods, func(i, j int) bool { return len(mods[i].Path) > len(mods[j].Path) }) @@ -307,9 +300,9 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // in the build list, and isn't in any other module that the user has // shimmed in via a "replace" directive. // Moreover, the import path is reserved for the standard library, so - // QueryPackage cannot possibly find a module containing this package. + // QueryPattern cannot possibly find a module containing this package. // - // Instead of trying QueryPackage, report an ImportMissingError immediately. + // Instead of trying QueryPattern, report an ImportMissingError immediately. return module.Version{}, &ImportMissingError{Path: path} } @@ -328,11 +321,11 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(ctx, path, "latest", CheckAllowed) + candidates, err := QueryPattern(ctx, path, "latest", CheckAllowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever - // low-level error QueryPackage produced. + // low-level error QueryPattern produced. return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} } else { return module.Version{}, err @@ -345,7 +338,7 @@ func queryImport(ctx context.Context, path string) (module.Version, error) { canAdd := true for _, bm := range buildList { if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { - // QueryPackage proposed that we add module cm to provide the package, + // QueryPattern proposed that we add module cm to provide the package, // but we already depend on a newer version of that module (and we don't // have the package). // diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 1f50dcb11c..3344242489 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -22,7 +22,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/load" "cmd/go/internal/lockedfile" "cmd/go/internal/modconv" "cmd/go/internal/modfetch" @@ -35,8 +34,7 @@ import ( ) var ( - mustUseModules = false - initialized bool + initialized bool modRoot string Target module.Version @@ -55,17 +53,42 @@ var ( CmdModInit bool // running 'go mod init' CmdModModule string // module argument for 'go mod init' + // RootMode determines whether a module root is needed. + RootMode Root + + // ForceUseModules may be set to force modules to be enabled when + // GO111MODULE=auto or to report an error when GO111MODULE=off. + ForceUseModules bool + allowMissingModuleImports bool ) +type Root int + +const ( + // AutoRoot is the default for most commands. modload.Init will look for + // a go.mod file in the current directory or any parent. If none is found, + // modules may be disabled (GO111MODULE=on) or commands may run in a + // limited module mode. + AutoRoot Root = iota + + // NoRoot is used for commands that run in module mode and ignore any go.mod + // file the current directory or in parent directories. + NoRoot + + // NeedRoot is used for commands that must run in module mode and don't + // make sense without a main module. + NeedRoot +) + // ModFile returns the parsed go.mod file. // -// Note that after calling ImportPaths or LoadBuildList, +// Note that after calling LoadPackages or LoadAllModules, // the require statements in the modfile.File are no longer // the source of truth and will be ignored: edits made directly // will be lost at the next call to WriteGoMod. // To make permanent changes to the require statements -// in go.mod, edit it before calling ImportPaths or LoadBuildList. +// in go.mod, edit it before loading. func ModFile() *modfile.File { Init() if modFile == nil { @@ -92,15 +115,19 @@ func Init() { // Keep in sync with WillBeEnabled. We perform extra validation here, and // there are lots of diagnostics and side effects, so we can't use // WillBeEnabled directly. + var mustUseModules bool env := cfg.Getenv("GO111MODULE") switch env { default: base.Fatalf("go: unknown environment setting GO111MODULE=%s", env) - case "auto", "": - mustUseModules = false - case "on": + case "auto": + mustUseModules = ForceUseModules + case "on", "": mustUseModules = true case "off": + if ForceUseModules { + base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") + } mustUseModules = false return } @@ -135,12 +162,20 @@ func Init() { if CmdModInit { // Running 'go mod init': go.mod will be created in current directory. modRoot = base.Cwd + } else if RootMode == NoRoot { + if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { + base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") + } + modRoot = "" } else { modRoot = findModuleRoot(base.Cwd) if modRoot == "" { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } + if RootMode == NeedRoot { + base.Fatalf("go: cannot find main module; see 'go help modules'") + } if !mustUseModules { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. @@ -154,6 +189,9 @@ func Init() { // when it happens. See golang.org/issue/26708. modRoot = "" fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) + if !mustUseModules { + return + } } } if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { @@ -172,14 +210,6 @@ func Init() { } cfg.ModulesEnabled = true - load.ModBinDir = BinDir - load.ModLookup = Lookup - load.ModPackageModuleInfo = PackageModuleInfo - load.ModImportPaths = ImportPaths - load.ModPackageBuildInfo = PackageBuildInfo - load.ModInfoProg = ModInfoProg - load.ModImportFromFiles = ImportFromFiles - load.ModDirImportPath = DirImportPath if modRoot == "" { // We're in module mode, but not inside a module. @@ -205,10 +235,6 @@ func Init() { } } -func init() { - load.ModInit = Init -} - // WillBeEnabled checks whether modules should be enabled but does not // initialize modules by installing hooks. If Init has already been called, // WillBeEnabled returns the same result as Enabled. @@ -219,10 +245,12 @@ func init() { // be called until the command is installed and flags are parsed. Instead of // calling Init and Enabled, the main package can call this function. func WillBeEnabled() bool { - if modRoot != "" || mustUseModules { + if modRoot != "" || cfg.ModulesEnabled { + // Already enabled. return true } if initialized { + // Initialized, not enabled. return false } @@ -230,9 +258,9 @@ func WillBeEnabled() bool { // exits, so it can't call this function directly. env := cfg.Getenv("GO111MODULE") switch env { - case "on": + case "on", "": return true - case "auto", "": + case "auto": break default: return false @@ -263,7 +291,7 @@ func WillBeEnabled() bool { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || mustUseModules + return modRoot != "" || cfg.ModulesEnabled } // ModRoot returns the root of the main module. @@ -531,7 +559,7 @@ func setDefaultBuildMod() { return } if modRoot == "" { - cfg.BuildMod = "mod" + cfg.BuildMod = "readonly" return } @@ -554,13 +582,7 @@ func setDefaultBuildMod() { cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) } - p := ModFilePath() - if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { - cfg.BuildMod = "readonly" - cfg.BuildModReason = "go.mod file is read-only." - return - } - cfg.BuildMod = "mod" + cfg.BuildMod = "readonly" } func legacyModInit() { @@ -858,10 +880,12 @@ func WriteGoMod() { if dirty && cfg.BuildMod == "readonly" { // If we're about to fail due to -mod=readonly, // prefer to report a dirty go.mod over a dirty go.sum - if cfg.BuildModReason != "" { - base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) - } else if cfg.BuildModExplicit { + if cfg.BuildModExplicit { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") + } else if cfg.BuildModReason != "" { + base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) + } else { + base.Fatalf("go: updates to go.mod needed; try 'go mod tidy' first") } } @@ -920,9 +944,9 @@ func WriteGoMod() { // keepSums returns a set of module sums to preserve in go.sum. The set // includes entries for all modules used to load packages (according to -// the last load function like ImportPaths, LoadALL, etc.). It also contains -// entries for go.mod files needed for MVS (the version of these entries -// ends with "/go.mod"). +// the last load function such as LoadPackages or ImportFromFiles). +// It also contains entries for go.mod files needed for MVS (the version +// of these entries ends with "/go.mod"). // // If addDirect is true, the set also includes sums for modules directly // required by go.mod, as represented by the index, with replacements applied. @@ -954,8 +978,7 @@ func keepSums(addDirect bool) map[module.Version]bool { } walk(Target) - // Add entries for modules that provided packages loaded with ImportPaths, - // LoadALL, or similar functions. + // Add entries for modules from which packages were loaded. if loaded != nil { for _, pkg := range loaded.pkgs { m := pkg.mod diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 1664d8c5be..9194f9cc7c 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -7,9 +7,9 @@ package modload // This file contains the module-mode package loader, as well as some accessory // functions pertaining to the package import graph. // -// There are several exported entry points into package loading (such as -// ImportPathsQuiet and LoadALL), but they are all implemented in terms of -// loadFromRoots, which itself manipulates an instance of the loader struct. +// There are two exported entry points into package loading — LoadPackages and +// ImportFromFiles — both implemented in terms of loadFromRoots, which itself +// manipulates an instance of the loader struct. // // Although most of the loading state is maintained in the loader struct, // one key piece - the build list - is a global, so that it can be modified @@ -24,13 +24,12 @@ package modload // // The first step of each iteration identifies a set of “root” packages. // Normally the root packages are exactly those matching the named pattern -// arguments. However, for the "all" meta-pattern and related functions -// (LoadALL, LoadVendor), the final set of packages is computed from the package -// import graph, and therefore cannot be an initial input to loading that graph. -// Instead, the root packages for the "all" pattern are those contained in the -// main module, and allPatternIsRoot parameter to the loader instructs it to -// dynamically expand those roots to the full "all" pattern as loading -// progresses. +// arguments. However, for the "all" meta-pattern, the final set of packages is +// computed from the package import graph, and therefore cannot be an initial +// input to loading that graph. Instead, the root packages for the "all" pattern +// are those contained in the main module, and allPatternIsRoot parameter to the +// loader instructs it to dynamically expand those roots to the full "all" +// pattern as loading progresses. // // The pkgInAll flag on each loadPkg instance tracks whether that // package is known to match the "all" meta-pattern. @@ -126,25 +125,57 @@ import ( // It holds details about individual packages. var loaded *loader -// ImportPaths returns the set of packages matching the args (patterns), -// on the target platform. Modules may be added to the build list -// to satisfy new imports. -func ImportPaths(ctx context.Context, patterns []string) []*search.Match { - matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) - search.WarnUnmatched(matches) - return matches +// PackageOpts control the behavior of the LoadPackages function. +type PackageOpts struct { + // Tags are the build tags in effect (as interpreted by the + // cmd/go/internal/imports package). + // If nil, treated as equivalent to imports.Tags(). + Tags map[string]bool + + // ResolveMissingImports indicates that we should attempt to add module + // dependencies as needed to resolve imports of packages that are not found. + // + // For commands that support the -mod flag, resolving imports may still fail + // if the flag is set to "readonly" (the default) or "vendor". + ResolveMissingImports bool + + // LoadTests loads the test dependencies of each package matching a requested + // pattern. If ResolveMissingImports is also true, test dependencies will be + // resolved if missing. + LoadTests bool + + // UseVendorAll causes the "all" package pattern to be interpreted as if + // running "go mod vendor" (or building with "-mod=vendor"). + // + // Once lazy loading is implemented, this will be a no-op for modules that + // declare 'go 1.16' or higher. + UseVendorAll bool + + // AllowErrors indicates that LoadPackages should not terminate the process if + // an error occurs. + AllowErrors bool + + // SilenceErrors indicates that LoadPackages should not print errors + // that occur while loading packages. SilenceErrors implies AllowErrors. + SilenceErrors bool + + // SilenceUnmatchedWarnings suppresses the warnings normally emitted for + // patterns that did not match any packages. + SilenceUnmatchedWarnings bool } -// ImportPathsQuiet is like ImportPaths but does not warn about patterns with -// no matches. It also lets the caller specify a set of build tags to match -// packages. The build tags should typically be imports.Tags() or -// imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { +// LoadPackages identifies the set of packages matching the given patterns and +// loads the packages in the import graph rooted at that set. +func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (matches []*search.Match, loadedPackages []string) { InitMod(ctx) + if opts.Tags == nil { + opts.Tags = imports.Tags() + } + patterns = search.CleanPatterns(patterns) + matches = make([]*search.Match, 0, len(patterns)) allPatternIsRoot := false - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { + for _, pattern := range patterns { matches = append(matches, search.NewMatch(pattern)) if pattern == "all" { allPatternIsRoot = true @@ -191,14 +222,14 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(ctx, m, tags, includeStd, buildList) + matchPackages(ctx, m, opts.Tags, includeStd, buildList) case m.Pattern() == "all": if ld == nil { // The initial roots are the packages in the main module. // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(ctx, m, tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -217,9 +248,12 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo } loaded = loadFromRoots(loaderParams{ - tags: tags, + tags: opts.Tags, + loadTests: opts.LoadTests, + resolveMissing: opts.ResolveMissingImports, + + allClosesOverTests: index.allPatternClosesOverTests() && !opts.UseVendorAll, allPatternIsRoot: allPatternIsRoot, - allClosesOverTests: index.allPatternClosesOverTests(), listRoots: func() (roots []string) { updateMatches(nil) @@ -232,10 +266,44 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo // One last pass to finalize wildcards. updateMatches(loaded) + + // Report errors, if any. checkMultiplePaths() - WriteGoMod() + for _, pkg := range loaded.pkgs { + if pkg.err != nil && !opts.SilenceErrors { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%s: %v\n", pkg.stackText(), pkg.err) + } else { + base.Errorf("%s: %v", pkg.stackText(), pkg.err) + } + } + if !pkg.isTest() { + loadedPackages = append(loadedPackages, pkg.path) + } + } + if !opts.SilenceErrors { + // Also list errors in matching patterns (such as directory permission + // errors for wildcard patterns). + for _, match := range matches { + for _, err := range match.Errs { + if opts.AllowErrors { + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + base.Errorf("%v", err) + } + } + } + } + base.ExitIfErrors() + + if !opts.SilenceUnmatchedWarnings { + search.WarnUnmatched(matches) + } - return matches + // Success! Update go.mod (if needed) and return the results. + WriteGoMod() + sort.Strings(loadedPackages) + return matches, loadedPackages } // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories @@ -425,13 +493,14 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { } loaded = loadFromRoots(loaderParams{ - tags: tags, + tags: tags, + resolveMissing: true, + allClosesOverTests: index.allPatternClosesOverTests(), listRoots: func() (roots []string) { roots = append(roots, imports...) roots = append(roots, testImports...) return roots }, - allClosesOverTests: index.allPatternClosesOverTests(), }) WriteGoMod() } @@ -462,58 +531,6 @@ func DirImportPath(dir string) string { return "." } -// LoadALL returns the set of all packages in the current module -// and their dependencies in any other modules, without filtering -// due to build tags, except "+build ignore". -// It adds modules to the build list as needed to satisfy new imports. -// This set is useful for deciding whether a particular import is needed -// anywhere in a module. -// -// In modules that specify "go 1.16" or higher, ALL follows only one layer of -// test dependencies. In "go 1.15" or lower, ALL follows the imports of tests of -// dependencies of tests. -func LoadALL(ctx context.Context) []string { - InitMod(ctx) - return loadAll(ctx, index.allPatternClosesOverTests()) -} - -// LoadVendor is like LoadALL but only follows test dependencies -// for tests in the main module. Tests in dependency modules are -// ignored completely. -// This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor(ctx context.Context) []string { - InitMod(ctx) - // 'go mod vendor' has never followed test dependencies since Go 1.11. - const closeOverTests = false - return loadAll(ctx, closeOverTests) -} - -func loadAll(ctx context.Context, closeOverTests bool) []string { - inTarget := TargetPackages(ctx, "...") - loaded = loadFromRoots(loaderParams{ - tags: imports.AnyTags(), - listRoots: func() []string { return inTarget.Pkgs }, - allPatternIsRoot: true, - allClosesOverTests: closeOverTests, - }) - checkMultiplePaths() - WriteGoMod() - - var paths []string - for _, pkg := range loaded.pkgs { - if pkg.err != nil { - base.Errorf("%s: %v", pkg.stackText(), pkg.err) - continue - } - paths = append(paths, pkg.path) - } - for _, err := range inTarget.Errs { - base.Errorf("%v", err) - } - base.ExitIfErrors() - return paths -} - // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. @@ -631,14 +648,15 @@ type loader struct { } type loaderParams struct { - tags map[string]bool // tags for scanDir - listRoots func() []string - allPatternIsRoot bool // Is the "all" pattern an additional root? + tags map[string]bool // tags for scanDir + loadTests bool + resolveMissing bool + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? -} + allPatternIsRoot bool // Is the "all" pattern an additional root? -// LoadTests controls whether the loaders load tests of the root packages. -var LoadTests bool + listRoots func() []string +} func (ld *loader) reset() { select { @@ -791,6 +809,10 @@ func loadFromRoots(params loaderParams) *loader { ld.buildStacks() + if !ld.resolveMissing { + // We've loaded as much as we can without resolving missing imports. + break + } modAddedBy := ld.resolveMissingImports(addedModuleFor) if len(modAddedBy) == 0 { break @@ -958,7 +980,7 @@ func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { // also in "all" (as above). wantTest = true - case LoadTests && new.has(pkgIsRoot): + case ld.loadTests && new.has(pkgIsRoot): // LoadTest explicitly requests tests of “the root packages”. wantTest = true } @@ -996,6 +1018,14 @@ func (ld *loader) load(pkg *loadPkg) { return } + if search.IsMetaPackage(pkg.path) { + pkg.err = &invalidImportError{ + importPath: pkg.path, + err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), + } + return + } + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path) if pkg.dir == "" { return @@ -1106,8 +1136,12 @@ func (ld *loader) stdVendor(parentPath, path string) string { // Do the same for importers beginning with the prefix 'vendor/' even if we // are *inside* of the 'std' module: the 'vendor/' packages that resolve // globally from GOROOT/src/vendor (and are listed as part of 'go list std') - // are distinct from the real module dependencies, and cannot import internal - // packages from the real module. + // are distinct from the real module dependencies, and cannot import + // internal packages from the real module. + // + // (Note that although the 'vendor/' packages match the 'std' *package* + // pattern, they are not part of the std *module*, and do not affect + // 'go mod tidy' and similar module commands when working within std.) vendorPath := pathpkg.Join("vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath @@ -1248,7 +1282,7 @@ func (pkg *loadPkg) why() string { // Why returns the "go mod why" output stanza for the given package, // without the leading # comment. -// The package graph must have been loaded already, usually by LoadALL. +// The package graph must have been loaded already, usually by LoadPackages. // If there is no reason for the package to be in the current build, // Why returns an empty string. func Why(path string) string { diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 18dd293ac9..6457a7d968 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -122,12 +122,21 @@ func checkRetractions(ctx context.Context, m module.Version) error { // Load go.mod for that version. // If the version is replaced, we'll load retractions from the replacement. + // // If there's an error loading the go.mod, we'll return it here. // These errors should generally be ignored by callers of checkRetractions, // since they happen frequently when we're offline. These errors are not // equivalent to ErrDisallowed, so they may be distinguished from // retraction errors. - summary, err := goModSummary(module.Version{Path: path, Version: rev.Version}) + // + // We load the raw file here: the go.mod file may have a different module + // path that we expect if the module or its repository was renamed. + // We still want to apply retractions to other aliases of the module. + rm := module.Version{Path: path, Version: rev.Version} + if repl := Replacement(rm); repl.Path != "" { + rm = repl + } + summary, err := rawGoModSummary(rm) if err != nil { return &entry{err: err} } @@ -378,87 +387,90 @@ func goModSummary(m module.Version) (*modFileSummary, error) { panic("internal error: goModSummary called on the Target module") } - type cached struct { - summary *modFileSummary - err error - } - c := goModSummaryCache.Do(m, func() interface{} { - if cfg.BuildMod == "vendor" { - summary := &modFileSummary{ - module: module.Version{Path: m.Path}, - } - if vendorVersion[m.Path] != m.Version { - // This module is not vendored, so packages cannot be loaded from it and - // it cannot be relevant to the build. - return cached{summary, nil} - } + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return summary, nil + } - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() - // TODO(#36876): Load the "go" version from vendor/modules.txt and store it - // in rawGoVersion with the appropriate key. + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. - // We don't know what versions the vendored module actually relies on, - // so assume that it requires everything. - summary.require = vendorList - return cached{summary, nil} - } + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return summary, nil + } - actual := Replacement(m) - if actual.Path == "" { - actual = m - } - summary, err := rawGoModSummary(actual) - if err != nil { - return cached{nil, err} - } + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + summary, err := rawGoModSummary(actual) + if err != nil { + return nil, err + } - if actual.Version == "" { - // The actual module is a filesystem-local replacement, for which we have - // unfortunately not enforced any sort of invariants about module lines or - // matching module paths. Anything goes. - // - // TODO(bcmills): Remove this special-case, update tests, and add a - // release note. - } else { - if summary.module.Path == "" { - return cached{nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line"))} - } + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line")) + } - // In theory we should only allow mpath to be unequal to m.Path here if the - // version that we fetched lacks an explicit go.mod file: if the go.mod file - // is explicit, then it should match exactly (to ensure that imports of other - // packages within the module are interpreted correctly). Unfortunately, we - // can't determine that information from the module proxy protocol: we'll have - // to leave that validation for when we load actual packages from within the - // module. - if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { - return cached{nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: module declares its path as: %s - but was required as: %s`, mpath, m.Path))} - } + but was required as: %s`, mpath, m.Path)) } + } - if index != nil && len(index.exclude) > 0 { - // Drop any requirements on excluded versions. - nonExcluded := summary.require[:0] + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false + for _, r := range summary.require { + if index.exclude[r] { + haveExcludedReqs = true + break + } + } + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) for _, r := range summary.require { if !index.exclude[r] { - nonExcluded = append(nonExcluded, r) + s.require = append(s.require, r) } } - summary.require = nonExcluded + summary = s } - return cached{summary, nil} - }).(cached) - - return c.summary, c.err + } + return summary, nil } -var goModSummaryCache par.Cache // module.Version → goModSummary result - // rawGoModSummary returns a new summary of the go.mod file for module m, // ignoring all replacements that may apply to m and excludes that may apply to // its dependencies. @@ -469,62 +481,72 @@ func rawGoModSummary(m module.Version) (*modFileSummary, error) { panic("internal error: rawGoModSummary called on the Target module") } - summary := new(modFileSummary) - var f *modfile.File - if m.Version == "" { - // m is a replacement module with only a file path. - dir := m.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") + type cached struct { + summary *modFileSummary + err error + } + c := rawGoModSummaryCache.Do(m, func() interface{} { + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") - data, err := lockedfile.Read(gomod) - if err != nil { - return nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err)) - } - f, err = modfile.ParseLax(gomod, data, nil) - if err != nil { - return nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err)) - } - } else { - if !semver.IsValid(m.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + data, err := lockedfile.Read(gomod) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err))} + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err))} + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return cached{nil, err} + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return cached{nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err))} + } } - data, err := modfetch.GoMod(m.Path, m.Version) - if err != nil { - return nil, err + if f.Module != nil { + summary.module = f.Module.Mod } - f, err = modfile.ParseLax("go.mod", data, nil) - if err != nil { - return nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err)) + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version } - } - - if f.Module != nil { - summary.module = f.Module.Mod - } - if f.Go != nil && f.Go.Version != "" { - rawGoVersion.LoadOrStore(m, f.Go.Version) - summary.goVersionV = "v" + f.Go.Version - } - if len(f.Require) > 0 { - summary.require = make([]module.Version, 0, len(f.Require)) - for _, req := range f.Require { - summary.require = append(summary.require, req.Mod) + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } } - } - if len(f.Retract) > 0 { - summary.retract = make([]retraction, 0, len(f.Retract)) - for _, ret := range f.Retract { - summary.retract = append(summary.retract, retraction{ - VersionInterval: ret.VersionInterval, - Rationale: ret.Rationale, - }) + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } } - } - return summary, nil + return cached{summary, nil} + }).(cached) + + return c.summary, c.err } + +var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index f67a738677..e75d901ec6 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -212,7 +212,20 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed default: // Direct lookup of semantic version or commit identifier. - // + + // If the query is a valid semantic version and that version is replaced, + // use the replacement module without searching the proxy. + canonicalQuery := module.CanonicalVersion(query) + if canonicalQuery != "" { + m := module.Version{Path: path, Version: query} + if r := Replacement(m); r.Path != "" { + if err := allowed(ctx, m); errors.Is(err, ErrDisallowed) { + return nil, err + } + return &modfetch.RevInfo{Version: query}, nil + } + } + // If the identifier is not a canonical semver tag — including if it's a // semver tag with a +metadata suffix — then modfetch.Stat will populate // info.Version with a suitable pseudo-version. @@ -222,9 +235,9 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // The full query doesn't correspond to a tag. If it is a semantic version // with a +metadata suffix, see if there is a tag without that suffix: // semantic versioning defines them to be equivalent. - if vers := module.CanonicalVersion(query); vers != "" && vers != query { - info, err = modfetch.Stat(proxy, path, vers) - if !errors.Is(err, os.ErrNotExist) { + if canonicalQuery != "" && query != canonicalQuery { + info, err = modfetch.Stat(proxy, path, canonicalQuery) + if err != nil && !errors.Is(err, os.ErrNotExist) { return info, err } } @@ -418,20 +431,6 @@ type QueryResult struct { Packages []string } -// QueryPackage looks up the module(s) containing path at a revision matching -// query. The results are sorted by module path length in descending order. -// -// If the package is in the main module, QueryPackage considers only the main -// module and only the version "latest", without checking for other possible -// modules. -func QueryPackage(ctx context.Context, path, query string, allowed AllowedFunc) ([]QueryResult, error) { - m := search.NewMatch(path) - if m.IsLocal() || !m.IsLiteral() { - return nil, fmt.Errorf("pattern %s is not an importable package", path) - } - return QueryPattern(ctx, path, query, allowed) -} - // QueryPattern looks up the module(s) containing at least one package matching // the given pattern at the given version. The results are sorted by module path // length in descending order. diff --git a/src/cmd/go/internal/str/str_test.go b/src/cmd/go/internal/str/str_test.go new file mode 100644 index 0000000000..147ce1a63e --- /dev/null +++ b/src/cmd/go/internal/str/str_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package str + +import "testing" + +var foldDupTests = []struct { + list []string + f1, f2 string +}{ + {StringList("math/rand", "math/big"), "", ""}, + {StringList("math", "strings"), "", ""}, + {StringList("strings"), "", ""}, + {StringList("strings", "strings"), "strings", "strings"}, + {StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, +} + +func TestFoldDup(t *testing.T) { + for _, tt := range foldDupTests { + f1, f2 := FoldDup(tt.list) + if f1 != tt.f1 || f2 != tt.f2 { + t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) + } + } +} diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 1ea6d2881e..51d333d866 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -29,7 +29,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" - "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/trace" "cmd/go/internal/work" @@ -568,7 +567,7 @@ var defaultVetFlags = []string{ } func runTest(ctx context.Context, cmd *base.Command, args []string) { - modload.LoadTests = true + load.ModResolveTests = true pkgArgs, testArgs = testFlags(args) diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 4f0a8924f1..d2671ff5a7 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -212,6 +212,10 @@ func testFlags(args []string) (packageNames, passToTest []string) { } }) + // firstUnknownFlag helps us report an error when flags not known to 'go + // test' are used along with -i or -c. + firstUnknownFlag := "" + explicitArgs := make([]string, 0, len(args)) inPkgList := false afterFlagWithoutValue := false @@ -288,6 +292,10 @@ func testFlags(args []string) (packageNames, passToTest []string) { break } + if firstUnknownFlag == "" { + firstUnknownFlag = nd.RawArg + } + explicitArgs = append(explicitArgs, nd.RawArg) args = remainingArgs if !nd.HasValue { @@ -312,6 +320,14 @@ func testFlags(args []string) (packageNames, passToTest []string) { args = remainingArgs } + if firstUnknownFlag != "" && (testC || cfg.BuildI) { + buildFlag := "-c" + if !testC { + buildFlag = "-i" + } + fmt.Fprintf(os.Stderr, "flag %s is not a 'go test' flag (unknown flags cannot be used with %s)\n", firstUnknownFlag, buildFlag) + exitWithUsage() + } var injectedFlags []string if testJSON { diff --git a/src/cmd/go/internal/get/discovery.go b/src/cmd/go/internal/vcs/discovery.go index afa6ef455f..327b44cb9a 100644 --- a/src/cmd/go/internal/get/discovery.go +++ b/src/cmd/go/internal/vcs/discovery.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/xml" diff --git a/src/cmd/go/internal/get/pkg_test.go b/src/cmd/go/internal/vcs/discovery_test.go index fc6a179c2e..eb99fdf64c 100644 --- a/src/cmd/go/internal/get/pkg_test.go +++ b/src/cmd/go/internal/vcs/discovery_test.go @@ -2,35 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( - "cmd/go/internal/str" "reflect" "strings" "testing" ) -var foldDupTests = []struct { - list []string - f1, f2 string -}{ - {str.StringList("math/rand", "math/big"), "", ""}, - {str.StringList("math", "strings"), "", ""}, - {str.StringList("strings"), "", ""}, - {str.StringList("strings", "strings"), "strings", "strings"}, - {str.StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, -} - -func TestFoldDup(t *testing.T) { - for _, tt := range foldDupTests { - f1, f2 := str.FoldDup(tt.list) - if f1 != tt.f1 || f2 != tt.f2 { - t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) - } - } -} - var parseMetaGoImportsTests = []struct { in string mod ModuleMode diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/vcs/vcs.go index 24c32935d0..90bf10244d 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/json" @@ -21,29 +21,28 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/load" "cmd/go/internal/web" ) // A vcsCmd describes how to use a version control system // like Mercurial, Git, or Subversion. -type vcsCmd struct { - name string - cmd string // name of binary to invoke command +type Cmd struct { + Name string + Cmd string // name of binary to invoke command - createCmd []string // commands to download a fresh copy of a repository - downloadCmd []string // commands to download updates into an existing repository + CreateCmd []string // commands to download a fresh copy of a repository + DownloadCmd []string // commands to download updates into an existing repository - tagCmd []tagCmd // commands to list tags - tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd []string // commands to sync to specific tag - tagSyncDefault []string // commands to sync to default tag + TagCmd []tagCmd // commands to list tags + TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + TagSyncCmd []string // commands to sync to specific tag + TagSyncDefault []string // commands to sync to default tag - scheme []string - pingCmd string + Scheme []string + PingCmd string - remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) - resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) + RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) + ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) } var defaultSecureScheme = map[string]bool{ @@ -54,7 +53,7 @@ var defaultSecureScheme = map[string]bool{ "ssh": true, } -func (v *vcsCmd) isSecure(repo string) bool { +func (v *Cmd) IsSecure(repo string) bool { u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. @@ -63,8 +62,8 @@ func (v *vcsCmd) isSecure(repo string) bool { return v.isSecureScheme(u.Scheme) } -func (v *vcsCmd) isSecureScheme(scheme string) bool { - switch v.cmd { +func (v *Cmd) isSecureScheme(scheme string) bool { + switch v.Cmd { case "git": // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a // colon-separated list of schemes that are allowed to be used with git @@ -89,7 +88,7 @@ type tagCmd struct { } // vcsList lists the known version control systems -var vcsList = []*vcsCmd{ +var vcsList = []*Cmd{ vcsHg, vcsGit, vcsSvn, @@ -97,11 +96,15 @@ var vcsList = []*vcsCmd{ vcsFossil, } +// vcsMod is a stub for the "mod" scheme. It's returned by +// repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. +var vcsMod = &Cmd{Name: "mod"} + // vcsByCmd returns the version control system for the given // command name (hg, git, svn, bzr). -func vcsByCmd(cmd string) *vcsCmd { +func vcsByCmd(cmd string) *Cmd { for _, vcs := range vcsList { - if vcs.cmd == cmd { + if vcs.Cmd == cmd { return vcs } } @@ -109,31 +112,31 @@ func vcsByCmd(cmd string) *vcsCmd { } // vcsHg describes how to use Mercurial. -var vcsHg = &vcsCmd{ - name: "Mercurial", - cmd: "hg", +var vcsHg = &Cmd{ + Name: "Mercurial", + Cmd: "hg", - createCmd: []string{"clone -U -- {repo} {dir}"}, - downloadCmd: []string{"pull"}, + CreateCmd: []string{"clone -U -- {repo} {dir}"}, + DownloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' // for selecting a version. This lets people have // a go.release.r60 branch and a go1 branch // and make changes in both, without constantly // editing .hgtags. - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ {"tags", `^(\S+)`}, {"branches", `^(\S+)`}, }, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update default"}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update default"}, - scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify -- {scheme}://{repo}", - remoteRepo: hgRemoteRepo, + Scheme: []string{"https", "http", "ssh"}, + PingCmd: "identify -- {scheme}://{repo}", + RemoteRepo: hgRemoteRepo, } -func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { +func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsHg.runOutput(rootDir, "paths default") if err != nil { return "", err @@ -142,45 +145,45 @@ func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) } // vcsGit describes how to use Git. -var vcsGit = &vcsCmd{ - name: "Git", - cmd: "git", +var vcsGit = &Cmd{ + Name: "Git", + Cmd: "git", - createCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, - downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ // tags/xxx matches a git tag named xxx // origin/xxx matches a git branch named xxx on the default remote repository {"show-ref", `(?:tags|origin)/(\S+)$`}, }, - tagLookupCmd: []tagCmd{ + TagLookupCmd: []tagCmd{ {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, }, - tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, // both createCmd and downloadCmd update the working dir. // No need to do more here. We used to 'checkout master' // but that doesn't work if the default branch is not named master. // DO NOT add 'checkout master' here. // See golang.org/issue/9032. - tagSyncDefault: []string{"submodule update --init --recursive"}, + TagSyncDefault: []string{"submodule update --init --recursive"}, - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not // support such a separator for that command, and this use should be safe // without it because the {scheme} value comes from the predefined list above. // See golang.org/issue/33836. - pingCmd: "ls-remote {scheme}://{repo}", + PingCmd: "ls-remote {scheme}://{repo}", - remoteRepo: gitRemoteRepo, + RemoteRepo: gitRemoteRepo, } // scpSyntaxRe matches the SCP-like addresses used by Git to access // repositories by SSH. var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) -func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { +func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { cmd := "config remote.origin.url" errParse := errors.New("unable to parse output of git " + cmd) errRemoteOriginNotFound := errors.New("remote origin not found") @@ -216,7 +219,7 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error // Iterate over insecure schemes too, because this function simply // reports the state of the repo. If we can't see insecure schemes then // we can't report the actual repo URL. - for _, s := range vcsGit.scheme { + for _, s := range vcsGit.Scheme { if repoURL.Scheme == s { return repoURL.String(), nil } @@ -225,27 +228,27 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } // vcsBzr describes how to use Bazaar. -var vcsBzr = &vcsCmd{ - name: "Bazaar", - cmd: "bzr", +var vcsBzr = &Cmd{ + Name: "Bazaar", + Cmd: "bzr", - createCmd: []string{"branch -- {repo} {dir}"}, + CreateCmd: []string{"branch -- {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: []string{"pull --overwrite"}, + DownloadCmd: []string{"pull --overwrite"}, - tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update -r revno:-1"}, + TagCmd: []tagCmd{{"tags", `^(\S+)`}}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update -r revno:-1"}, - scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: bzrRemoteRepo, - resolveRepo: bzrResolveRepo, + Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: bzrRemoteRepo, + ResolveRepo: bzrResolveRepo, } -func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { +func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "config parent_location") if err != nil { return "", err @@ -253,7 +256,7 @@ func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error return strings.TrimSpace(string(outb)), nil } -func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { +func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) if err != nil { return "", err @@ -287,22 +290,22 @@ func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string } // vcsSvn describes how to use Subversion. -var vcsSvn = &vcsCmd{ - name: "Subversion", - cmd: "svn", +var vcsSvn = &Cmd{ + Name: "Subversion", + Cmd: "svn", - createCmd: []string{"checkout -- {repo} {dir}"}, - downloadCmd: []string{"update"}, + CreateCmd: []string{"checkout -- {repo} {dir}"}, + DownloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. - scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: svnRemoteRepo, + Scheme: []string{"https", "http", "svn", "svn+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: svnRemoteRepo, } -func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { +func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsSvn.runOutput(rootDir, "info") if err != nil { return "", err @@ -337,22 +340,22 @@ func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error const fossilRepoName = ".fossil" // vcsFossil describes how to use Fossil (fossil-scm.org) -var vcsFossil = &vcsCmd{ - name: "Fossil", - cmd: "fossil", +var vcsFossil = &Cmd{ + Name: "Fossil", + Cmd: "fossil", - createCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, - downloadCmd: []string{"up"}, + CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + DownloadCmd: []string{"up"}, - tagCmd: []tagCmd{{"tag ls", `(.*)`}}, - tagSyncCmd: []string{"up tag:{tag}"}, - tagSyncDefault: []string{"up trunk"}, + TagCmd: []tagCmd{{"tag ls", `(.*)`}}, + TagSyncCmd: []string{"up tag:{tag}"}, + TagSyncDefault: []string{"up trunk"}, - scheme: []string{"https", "http"}, - remoteRepo: fossilRemoteRepo, + Scheme: []string{"https", "http"}, + RemoteRepo: fossilRemoteRepo, } -func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err error) { +func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsFossil.runOutput(rootDir, "remote-url") if err != nil { return "", err @@ -360,8 +363,8 @@ func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err return strings.TrimSpace(string(out)), nil } -func (v *vcsCmd) String() string { - return v.name +func (v *Cmd) String() string { + return v.Name } // run runs the command line cmd in the given directory. @@ -371,24 +374,24 @@ func (v *vcsCmd) String() string { // If an error occurs, run prints the command line and the // command's combined stdout+stderr to standard error. // Otherwise run discards the command's output. -func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { +func (v *Cmd) run(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, true) return err } // runVerboseOnly is like run but only generates error output to standard error in verbose mode. -func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { +func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, false) return err } // runOutput is like run but returns the output of the command. -func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { +func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { return v.run1(dir, cmd, keyval, true) } // run1 is the generalized implementation of run and runOutput. -func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { +func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { m := make(map[string]string) for i := 0; i < len(keyval); i += 2 { m[keyval[i]] = keyval[i+1] @@ -420,25 +423,25 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) args = args[2:] } - _, err := exec.LookPath(v.cmd) + _, err := exec.LookPath(v.Cmd) if err != nil { fmt.Fprintf(os.Stderr, "go: missing %s command. See https://golang.org/s/gogetcmd\n", - v.name) + v.Name) return nil, err } - cmd := exec.Command(v.cmd, args...) + cmd := exec.Command(v.Cmd, args...) cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) if cfg.BuildX { fmt.Fprintf(os.Stderr, "cd %s\n", dir) - fmt.Fprintf(os.Stderr, "%s %s\n", v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) } out, err := cmd.Output() if err != nil { if verbose || cfg.BuildV { - fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { os.Stderr.Write(ee.Stderr) } else { @@ -449,15 +452,15 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) return out, err } -// ping pings to determine scheme to use. -func (v *vcsCmd) ping(scheme, repo string) error { - return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) +// Ping pings to determine scheme to use. +func (v *Cmd) Ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo) } -// create creates a new copy of repo in dir. +// Create creates a new copy of repo in dir. // The parent of dir must exist; dir must not. -func (v *vcsCmd) create(dir, repo string) error { - for _, cmd := range v.createCmd { +func (v *Cmd) Create(dir, repo string) error { + for _, cmd := range v.CreateCmd { if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { return err } @@ -465,9 +468,9 @@ func (v *vcsCmd) create(dir, repo string) error { return nil } -// download downloads any new changes for the repo in dir. -func (v *vcsCmd) download(dir string) error { - for _, cmd := range v.downloadCmd { +// Download downloads any new changes for the repo in dir. +func (v *Cmd) Download(dir string) error { + for _, cmd := range v.DownloadCmd { if err := v.run(dir, cmd); err != nil { return err } @@ -475,10 +478,10 @@ func (v *vcsCmd) download(dir string) error { return nil } -// tags returns the list of available tags for the repo in dir. -func (v *vcsCmd) tags(dir string) ([]string, error) { +// Tags returns the list of available tags for the repo in dir. +func (v *Cmd) Tags(dir string) ([]string, error) { var tags []string - for _, tc := range v.tagCmd { + for _, tc := range v.TagCmd { out, err := v.runOutput(dir, tc.cmd) if err != nil { return nil, err @@ -493,12 +496,12 @@ func (v *vcsCmd) tags(dir string) ([]string, error) { // tagSync syncs the repo in dir to the named tag, // which either is a tag returned by tags or is v.tagDefault. -func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == nil { +func (v *Cmd) TagSync(dir, tag string) error { + if v.TagSyncCmd == nil { return nil } if tag != "" { - for _, tc := range v.tagLookupCmd { + for _, tc := range v.TagLookupCmd { out, err := v.runOutput(dir, tc.cmd, "tag", tag) if err != nil { return err @@ -512,8 +515,8 @@ func (v *vcsCmd) tagSync(dir, tag string) error { } } - if tag == "" && v.tagSyncDefault != nil { - for _, cmd := range v.tagSyncDefault { + if tag == "" && v.TagSyncDefault != nil { + for _, cmd := range v.TagSyncDefault { if err := v.run(dir, cmd); err != nil { return err } @@ -521,7 +524,7 @@ func (v *vcsCmd) tagSync(dir, tag string) error { return nil } - for _, cmd := range v.tagSyncCmd { + for _, cmd := range v.TagSyncCmd { if err := v.run(dir, cmd, "tag", tag); err != nil { return err } @@ -540,11 +543,11 @@ type vcsPath struct { schemelessRepo bool // if true, the repo pattern lacks a scheme } -// vcsFromDir inspects dir and its parents to determine the +// FromDir inspects dir and its parents to determine the // version control system and code repository to use. // On return, root is the import path // corresponding to the root of the repository. -func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { +func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { // Clean and double-check that dir is in (a subdirectory of) srcRoot. dir = filepath.Clean(dir) srcRoot = filepath.Clean(srcRoot) @@ -552,13 +555,13 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } - var vcsRet *vcsCmd + var vcsRet *Cmd var rootRet string origDir := dir for len(dir) > len(srcRoot) { for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { root := filepath.ToSlash(dir[len(srcRoot)+1:]) // Record first VCS we find, but keep looking, // to detect mistakes like one kind of VCS inside another. @@ -568,12 +571,12 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { continue } // Allow .git inside .git, which can arise due to submodules. - if vcsRet == vcs && vcs.cmd == "git" { + if vcsRet == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", - filepath.Join(srcRoot, rootRet), vcsRet.cmd, filepath.Join(srcRoot, root), vcs.cmd) + filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) } } @@ -593,9 +596,9 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) } -// checkNestedVCS checks for an incorrectly-nested VCS-inside-VCS +// CheckNested checks for an incorrectly-nested VCS-inside-VCS // situation for dir, checking parents up until srcRoot. -func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { +func CheckNested(vcs *Cmd, dir, srcRoot string) error { if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } @@ -603,17 +606,17 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { otherDir := dir for len(otherDir) > len(srcRoot) { for _, otherVCS := range vcsList { - if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { // Allow expected vcs in original dir. if otherDir == dir && otherVCS == vcs { continue } // Allow .git inside .git, which can arise due to submodules. - if otherVCS == vcs && vcs.cmd == "git" { + if otherVCS == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. - return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.cmd, otherDir, otherVCS.cmd) + return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) } } // Move to parent. @@ -633,9 +636,7 @@ type RepoRoot struct { Repo string // repository URL, including scheme Root string // import path corresponding to root of repo IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) - VCS string // vcs type ("mod", "git", ...) - - vcs *vcsCmd // internal: vcs command access + VCS *Cmd } func httpPrefix(s string) string { @@ -662,7 +663,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur if err == errUnknownSite { rr, err = repoRootForImportDynamic(importPath, mod, security) if err != nil { - err = load.ImportErrorf(importPath, "unrecognized import path %q: %v", importPath, err) + err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err) } } if err != nil { @@ -677,7 +678,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { // Do not allow wildcards in the repo root. rr = nil - err = load.ImportErrorf(importPath, "cannot expand ... in %q", importPath) + err = importErrorf(importPath, "cannot expand ... in %q", importPath) } return rr, err } @@ -701,7 +702,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths m := srv.regexp.FindStringSubmatch(importPath) if m == nil { if srv.prefix != "" { - return nil, load.ImportErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath) + return nil, importErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath) } continue } @@ -735,15 +736,15 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if !srv.schemelessRepo { repoURL = match["repo"] } else { - scheme := vcs.scheme[0] // default to first scheme + scheme := vcs.Scheme[0] // default to first scheme repo := match["repo"] - if vcs.pingCmd != "" { + if vcs.PingCmd != "" { // If we know how to test schemes, scan to find one. - for _, s := range vcs.scheme { + for _, s := range vcs.Scheme { if security == web.SecureOnly && !vcs.isSecureScheme(s) { continue } - if vcs.ping(s, repo) == nil { + if vcs.Ping(s, repo) == nil { scheme = s break } @@ -754,8 +755,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths rr := &RepoRoot{ Repo: repoURL, Root: match["root"], - VCS: vcs.cmd, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -846,17 +846,21 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if err := validateRepoRoot(mmi.RepoRoot); err != nil { return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) } - vcs := vcsByCmd(mmi.VCS) - if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + var vcs *Cmd + if mmi.VCS == "mod" { + vcs = vcsMod + } else { + vcs = vcsByCmd(mmi.VCS) + if vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + } } rr := &RepoRoot{ Repo: mmi.RepoRoot, Root: mmi.Prefix, IsCustom: true, - VCS: mmi.VCS, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -1103,7 +1107,7 @@ var vcsPathsAfterDynamic = []*vcsPath{ func noVCSSuffix(match map[string]string) error { repo := match["repo"] for _, vcs := range vcsList { - if strings.HasSuffix(repo, "."+vcs.cmd) { + if strings.HasSuffix(repo, "."+vcs.Cmd) { return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) } } @@ -1133,7 +1137,7 @@ func bitbucketVCS(match map[string]string) error { // VCS it uses. See issue 5375. root := match["root"] for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).ping("https", root) == nil { + if vcsByCmd(vcs).Ping("https", root) == nil { resp.SCM = vcs break } @@ -1180,3 +1184,32 @@ func launchpadVCS(match map[string]string) error { } return nil } + +// importError is a copy of load.importError, made to avoid a dependency cycle +// on cmd/go/internal/load. It just needs to satisfy load.ImportPathError. +type importError struct { + importPath string + err error +} + +func importErrorf(path, format string, args ...interface{}) error { + err := &importError{importPath: path, err: fmt.Errorf(format, args...)} + if errStr := err.Error(); !strings.Contains(errStr, path) { + panic(fmt.Sprintf("path %q not in error %q", path, errStr)) + } + return err +} + +func (e *importError) Error() string { + return e.err.Error() +} + +func (e *importError) Unwrap() error { + // Don't return e.err directly, since we're only wrapping an error if %w + // was passed to ImportErrorf. + return errors.Unwrap(e.err) +} + +func (e *importError) ImportPath() string { + return e.importPath +} diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/vcs/vcs_test.go index 195bc231eb..5b874204f1 100644 --- a/src/cmd/go/internal/get/vcs_test.go +++ b/src/cmd/go/internal/vcs/vcs_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "errors" @@ -28,7 +28,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "github.com/golang/groupcache", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://github.com/golang/groupcache", }, }, @@ -41,14 +41,14 @@ func TestRepoRootForImportPath(t *testing.T) { { "hub.jazz.net/git/user1/pkgname", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, { "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, @@ -89,7 +89,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "hub.jazz.net/git/user/pkg.name", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user/pkg.name", }, }, @@ -102,7 +102,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -112,14 +112,14 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift.git", }, }, { "git.openstack.org/openstack/swift/go/hummingbird", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -148,21 +148,21 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.apache.org/package-name.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name.git", }, }, { "git.apache.org/package-name_2.x.git/path/to/lib", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name_2.x.git", }, }, { "chiselapp.com/user/kyle/repository/fossilgg", &RepoRoot{ - vcs: vcsFossil, + VCS: vcsFossil, Repo: "https://chiselapp.com/user/kyle/repository/fossilgg", }, }, @@ -191,8 +191,8 @@ func TestRepoRootForImportPath(t *testing.T) { t.Errorf("RepoRootForImportPath(%q): %v", test.path, err) continue } - if got.vcs.name != want.vcs.name || got.Repo != want.Repo { - t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo) + if got.VCS.Name != want.VCS.Name || got.Repo != want.Repo { + t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.VCS, got.Repo, want.VCS, want.Repo) } } } @@ -206,7 +206,7 @@ func TestFromDir(t *testing.T) { defer os.RemoveAll(tempDir) for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) + dir := filepath.Join(tempDir, "example.com", vcs.Name, "."+vcs.Cmd) if j&1 == 0 { err := os.MkdirAll(dir, 0755) if err != nil { @@ -225,24 +225,24 @@ func TestFromDir(t *testing.T) { } want := RepoRoot{ - vcs: vcs, - Root: path.Join("example.com", vcs.name), + VCS: vcs, + Root: path.Join("example.com", vcs.Name), } var got RepoRoot - got.vcs, got.Root, err = vcsFromDir(dir, tempDir) + got.VCS, got.Root, err = FromDir(dir, tempDir) if err != nil { t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) continue } - if got.vcs.name != want.vcs.name || got.Root != want.Root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root) + if got.VCS.Name != want.VCS.Name || got.Root != want.Root { + t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.VCS, got.Root, want.VCS, want.Root) } } } func TestIsSecure(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -267,7 +267,7 @@ func TestIsSecure(t *testing.T) { } for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } @@ -276,7 +276,7 @@ func TestIsSecure(t *testing.T) { func TestIsSecureGitAllowProtocol(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -307,7 +307,7 @@ func TestIsSecureGitAllowProtocol(t *testing.T) { defer os.Unsetenv("GIT_ALLOW_PROTOCOL") os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index c2de8d326d..5aa0f8e7ed 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -54,7 +54,14 @@ var ( func runVersion(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { - if *versionM || *versionV { + // If any of this command's flags were passed explicitly, error + // out, because they only make sense with arguments. + // + // Don't error if the flags came from GOFLAGS, since that can be + // a reasonable use case. For example, imagine GOFLAGS=-v to + // turn "verbose mode" on for all Go commands, which should not + // break "go version". + if (!base.InGOFLAGS("-m") && *versionM) || (!base.InGOFLAGS("-v") && *versionV) { fmt.Fprintf(os.Stderr, "go version: flags can only be used with arguments\n") base.SetExitStatus(2) return diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index cf2c8d59e8..b1bf806e46 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -13,7 +13,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" - "cmd/go/internal/modload" "cmd/go/internal/trace" "cmd/go/internal/work" ) @@ -54,7 +53,7 @@ See also: go fmt, go fix. } func runVet(ctx context.Context, cmd *base.Command, args []string) { - modload.LoadTests = true + load.ModResolveTests = true vetFlags, pkgArgs := vetFlags(args) diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index e99982ed36..86423f118c 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -9,8 +9,10 @@ import ( "errors" "fmt" "go/build" + "internal/goroot" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -18,8 +20,13 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modfetch" + "cmd/go/internal/modload" "cmd/go/internal/search" "cmd/go/internal/trace" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) var CmdBuild = &base.Command{ @@ -46,8 +53,9 @@ serving only as a check that the packages can be built. The -o flag forces build to write the resulting executable or object to the named output file or directory, instead of the default behavior described -in the last two paragraphs. If the named output is a directory that exists, -then any resulting executables will be written to that directory. +in the last two paragraphs. If the named output is an existing directory or +ends with a slash or backslash, then any resulting executables +will be written to that directory. The -i flag installs the packages that are dependencies of the target. @@ -380,10 +388,13 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { } if cfg.BuildO != "" { - // If the -o name exists and is a directory, then + // If the -o name exists and is a directory or + // ends with a slash or backslash, then // write all main packages to that directory. // Otherwise require only a single package be built. - if fi, err := os.Stat(cfg.BuildO); err == nil && fi.IsDir() { + if fi, err := os.Stat(cfg.BuildO); (err == nil && fi.IsDir()) || + strings.HasSuffix(cfg.BuildO, "/") || + strings.HasSuffix(cfg.BuildO, string(os.PathSeparator)) { if !explicitO { base.Fatalf("go build: build output %q already exists and is a directory", cfg.BuildO) } @@ -440,6 +451,33 @@ variable, which defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH environment variable is not set. Executables in $GOROOT are installed in $GOROOT/bin or $GOTOOLDIR instead of $GOBIN. +If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, ignoring the go.mod file in the current +directory or any parent directory, if there is one. This is useful for +installing executables without affecting the dependencies of the main module. +To eliminate ambiguity about which module versions are used in the build, the +arguments must satisfy the following constraints: + +- Arguments must be package paths or package patterns (with "..." wildcards). + They must not be standard packages (like fmt), meta-patterns (std, cmd, + all), or relative or absolute file paths. +- All arguments must have the same version suffix. Different queries are not + allowed, even if they refer to the same version. +- All arguments must refer to packages in the same module at the same version. +- No module is considered the "main" module. If the module containing + packages named on the command line has a go.mod file, it must not contain + directives (replace and exclude) that would cause it to be interpreted + differently than if it were the main module. The module must not require + a higher version of itself. +- Package path arguments must refer to main packages. Pattern arguments + will only match main packages. + +If the arguments don't have version suffixes, "go install" may run in +module-aware mode or GOPATH mode, depending on the GO111MODULE environment +variable and the presence of a go.mod file. See 'go help modules' for details. +If module-aware mode is enabled, "go install" runs in the context of the main +module. + When module-aware mode is disabled, other packages are installed in the directory $GOPATH/pkg/$GOOS_$GOARCH. When module-aware mode is enabled, other packages are built and cached but not installed. @@ -510,6 +548,12 @@ func libname(args []string, pkgs []*load.Package) (string, error) { } func runInstall(ctx context.Context, cmd *base.Command, args []string) { + for _, arg := range args { + if strings.Contains(arg, "@") && !build.IsLocalImport(arg) && !filepath.IsAbs(arg) { + installOutsideModule(ctx, args) + return + } + } BuildInit() InstallPackages(ctx, args, load.PackagesForBuild(ctx, args)) } @@ -634,6 +678,158 @@ func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Packag } } +// installOutsideModule implements 'go install pkg@version'. It builds and +// installs one or more main packages in module mode while ignoring any go.mod +// in the current directory or parent directories. +// +// See golang.org/issue/40276 for details and rationale. +func installOutsideModule(ctx context.Context, args []string) { + modload.ForceUseModules = true + modload.RootMode = modload.NoRoot + modload.AllowMissingModuleImports() + modload.Init() + + // Check that the arguments satisfy syntactic constraints. + var version string + for _, arg := range args { + if i := strings.Index(arg, "@"); i >= 0 { + version = arg[i+1:] + if version == "" { + base.Fatalf("go install %s: version must not be empty", arg) + } + break + } + } + patterns := make([]string, len(args)) + for i, arg := range args { + if !strings.HasSuffix(arg, "@"+version) { + base.Errorf("go install %s: all arguments must have the same version (@%s)", arg, version) + continue + } + p := arg[:len(arg)-len(version)-1] + switch { + case build.IsLocalImport(p): + base.Errorf("go install %s: argument must be a package path, not a relative path", arg) + case filepath.IsAbs(p): + base.Errorf("go install %s: argument must be a package path, not an absolute path", arg) + case search.IsMetaPackage(p): + base.Errorf("go install %s: argument must be a package path, not a meta-package", arg) + case path.Clean(p) != p: + base.Errorf("go install %s: argument must be a clean package path", arg) + case !strings.Contains(p, "...") && search.IsStandardImportPath(p) && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, p): + base.Errorf("go install %s: argument must not be a package in the standard library", arg) + default: + patterns[i] = p + } + } + base.ExitIfErrors() + BuildInit() + + // Query the module providing the first argument, load its go.mod file, and + // check that it doesn't contain directives that would cause it to be + // interpreted differently if it were the main module. + // + // If multiple modules match the first argument, accept the longest match + // (first result). It's possible this module won't provide packages named by + // later arguments, and other modules would. Let's not try to be too + // magical though. + allowed := modload.CheckAllowed + if modload.IsRevisionQuery(version) { + // Don't check for retractions if a specific revision is requested. + allowed = nil + } + qrs, err := modload.QueryPattern(ctx, patterns[0], version, allowed) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + installMod := qrs[0].Mod + data, err := modfetch.GoMod(installMod.Path, installMod.Version) + if err != nil { + base.Fatalf("go install %s: %v", args[0], err) + } + f, err := modfile.Parse("go.mod", data, nil) + if err != nil { + base.Fatalf("go install %s: %s: %v", args[0], installMod, err) + } + directiveFmt := "go install %s: %s\n" + + "\tThe go.mod file for the module providing named packages contains one or\n" + + "\tmore %s directives. It must not contain directives that would cause\n" + + "\tit to be interpreted differently than if it were the main module." + if len(f.Replace) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "replace") + } + if len(f.Exclude) > 0 { + base.Fatalf(directiveFmt, args[0], installMod, "exclude") + } + + // Initialize the build list using a dummy main module that requires the + // module providing the packages on the command line. + target := module.Version{Path: "go-install-target"} + modload.SetBuildList([]module.Version{target, installMod}) + + // Load packages for all arguments. Ignore non-main packages. + // Print a warning if an argument contains "..." and matches no main packages. + // PackagesForBuild already prints warnings for patterns that don't match any + // packages, so be careful not to double print. + matchers := make([]func(string) bool, len(patterns)) + for i, p := range patterns { + if strings.Contains(p, "...") { + matchers[i] = search.MatchPattern(p) + } + } + + // TODO(golang.org/issue/40276): don't report errors loading non-main packages + // matched by a pattern. + pkgs := load.PackagesForBuild(ctx, patterns) + mainPkgs := make([]*load.Package, 0, len(pkgs)) + mainCount := make([]int, len(patterns)) + nonMainCount := make([]int, len(patterns)) + for _, pkg := range pkgs { + if pkg.Name == "main" { + mainPkgs = append(mainPkgs, pkg) + for i := range patterns { + if matchers[i] != nil && matchers[i](pkg.ImportPath) { + mainCount[i]++ + } + } + } else { + for i := range patterns { + if matchers[i] == nil && patterns[i] == pkg.ImportPath { + base.Errorf("go install: package %s is not a main package", pkg.ImportPath) + } else if matchers[i] != nil && matchers[i](pkg.ImportPath) { + nonMainCount[i]++ + } + } + } + } + base.ExitIfErrors() + for i, p := range patterns { + if matchers[i] != nil && mainCount[i] == 0 && nonMainCount[i] > 0 { + fmt.Fprintf(os.Stderr, "go: warning: %q matched no main packages\n", p) + } + } + + // Check that named packages are all provided by the same module. + for _, mod := range modload.LoadedModules() { + if mod.Path == installMod.Path && mod.Version != installMod.Version { + base.Fatalf("go install: %s: module requires a higher version of itself (%s)", installMod, mod.Version) + } + } + for _, pkg := range mainPkgs { + if pkg.Module == nil { + // Packages in std, cmd, and their vendored dependencies + // don't have this field set. + base.Errorf("go install: package %s not provided by module %s", pkg.ImportPath, installMod) + } else if pkg.Module.Path != installMod.Path || pkg.Module.Version != installMod.Version { + base.Errorf("go install: package %s provided by module %s@%s\n\tAll packages must be provided by the same module (%s).", pkg.ImportPath, pkg.Module.Path, pkg.Module.Version, installMod) + } + } + base.ExitIfErrors() + + // Build and install the packages. + InstallPackages(ctx, patterns, mainPkgs) +} + // ExecCmd is the command to use to run user binaries. // Normally it is empty, meaning run the binaries directly. // If cross-compiling and running on a remote system or diff --git a/src/cmd/go/internal/work/build_test.go b/src/cmd/go/internal/work/build_test.go index c33de2635d..afed0fba72 100644 --- a/src/cmd/go/internal/work/build_test.go +++ b/src/cmd/go/internal/work/build_test.go @@ -221,7 +221,7 @@ func pkgImportPath(pkgpath string) *load.Package { // See https://golang.org/issue/18878. func TestRespectSetgidDir(t *testing.T) { switch runtime.GOOS { - case "darwin": + case "darwin", "ios": if runtime.GOARCH == "arm64" { t.Skip("can't set SetGID bit with chmod on iOS") } diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index 6613b6fe3f..a3c9b1a2c1 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -110,15 +110,15 @@ func contentID(buildID string) string { // hashToString converts the hash h to a string to be recorded // in package archives and binaries as part of the build ID. -// We use the first 96 bits of the hash and encode it in base64, -// resulting in a 16-byte string. Because this is only used for +// We use the first 120 bits of the hash (5 chunks of 24 bits each) and encode +// it in base64, resulting in a 20-byte string. Because this is only used for // detecting the need to rebuild installed files (not for lookups -// in the object file cache), 96 bits are sufficient to drive the +// in the object file cache), 120 bits are sufficient to drive the // probability of a false "do not need to rebuild" decision to effectively zero. // We embed two different hashes in archives and four in binaries, -// so cutting to 16 bytes is a significant savings when build IDs are displayed. -// (16*4+3 = 67 bytes compared to 64*4+3 = 259 bytes for the -// more straightforward option of printing the entire h in hex). +// so cutting to 20 bytes is a significant savings when build IDs are displayed. +// (20*4+3 = 83 bytes compared to 64*4+3 = 259 bytes for the +// more straightforward option of printing the entire h in base64). func hashToString(h [cache.HashSize]byte) string { const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" const chunks = 5 @@ -425,7 +425,7 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) // It's important that the overall buildID be unlikely verging on impossible // to appear in the output by chance, but that should be taken care of by // the actionID half; if it also appeared in the input that would be like an - // engineered 96-bit partial SHA256 collision. + // engineered 120-bit partial SHA256 collision. a.actionID = actionHash actionID := hashToString(actionHash) if a.json != nil { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index d975c36306..51fc2b588d 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -31,6 +31,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/trace" ) @@ -692,7 +693,7 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { } if p.Internal.BuildInfo != "" && cfg.ModulesEnabled { - if err := b.writeFile(objdir+"_gomod_.go", load.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { + if err := b.writeFile(objdir+"_gomod_.go", modload.ModInfoProg(p.Internal.BuildInfo, cfg.BuildToolchainName == "gccgo")); err != nil { return err } gofiles = append(gofiles, objdir+"_gomod_.go") @@ -1178,8 +1179,13 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { key, val := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) - // The linker writes source file paths that say GOROOT_FINAL. - fmt.Fprintf(h, "GOROOT=%s\n", cfg.GOROOT_FINAL) + // The linker writes source file paths that say GOROOT_FINAL, but + // only if -trimpath is not specified (see ld() in gc.go). + gorootFinal := cfg.GOROOT_FINAL + if cfg.BuildTrimpath { + gorootFinal = trimPathGoRootFinal + } + fmt.Fprintf(h, "GOROOT=%s\n", gorootFinal) // GO_EXTLINK_ENABLED controls whether the external linker is used. fmt.Fprintf(h, "GO_EXTLINK_ENABLED=%s\n", cfg.Getenv("GO_EXTLINK_ENABLED")) @@ -2418,7 +2424,7 @@ func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []strin // On OS X, some of the compilers behave as if -fno-common // is always set, and the Mach-O linker in 6l/8l assumes this. // See https://golang.org/issue/3253. - if cfg.Goos == "darwin" { + if cfg.Goos == "darwin" || cfg.Goos == "ios" { a = append(a, "-fno-common") } diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 6031897f88..d76574932e 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -25,6 +25,9 @@ import ( "crypto/sha1" ) +// The 'path' used for GOROOT_FINAL when -trimpath is specified +const trimPathGoRootFinal = "go" + // The Go toolchain. type gcToolchain struct{} @@ -569,7 +572,7 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) env := []string{} if cfg.BuildTrimpath { - env = append(env, "GOROOT_FINAL=go") + env = append(env, "GOROOT_FINAL="+trimPathGoRootFinal) } return b.run(root, dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) } diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index f78020032c..b0d6133768 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -9,7 +9,7 @@ package work import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/load" + "cmd/go/internal/modload" "cmd/internal/objabi" "cmd/internal/sys" "flag" @@ -21,7 +21,7 @@ import ( ) func BuildInit() { - load.ModInit() + modload.Init() instrumentInit() buildModeInit() @@ -121,7 +121,7 @@ func buildModeInit() { codegenArg = "-fPIC" } else { switch cfg.Goos { - case "darwin": + case "darwin", "ios": switch cfg.Goarch { case "arm64": codegenArg = "-shared" @@ -157,7 +157,7 @@ func buildModeInit() { ldBuildmode = "pie" case "windows": ldBuildmode = "pie" - case "darwin": + case "darwin", "ios": switch cfg.Goarch { case "arm64": codegenArg = "-shared" @@ -254,34 +254,18 @@ func buildModeInit() { case "": // Behavior will be determined automatically, as if no flag were passed. case "readonly", "vendor", "mod": - if !cfg.ModulesEnabled && !inGOFLAGS("-mod") { + if !cfg.ModulesEnabled && !base.InGOFLAGS("-mod") { base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) } default: base.Fatalf("-mod=%s not supported (can be '', 'mod', 'readonly', or 'vendor')", cfg.BuildMod) } if !cfg.ModulesEnabled { - if cfg.ModCacheRW && !inGOFLAGS("-modcacherw") { + if cfg.ModCacheRW && !base.InGOFLAGS("-modcacherw") { base.Fatalf("build flag -modcacherw only valid when using modules") } - if cfg.ModFile != "" && !inGOFLAGS("-mod") { + if cfg.ModFile != "" && !base.InGOFLAGS("-mod") { base.Fatalf("build flag -modfile only valid when using modules") } } } - -func inGOFLAGS(flag string) bool { - for _, goflag := range base.GOFLAGS() { - name := goflag - if strings.HasPrefix(name, "--") { - name = name[1:] - } - if i := strings.Index(name, "="); i >= 0 { - name = name[:i] - } - if name == flag { - return true - } - } - return false -} |