From ab2a5b48665eed6d670d719cdef5335bc3602359 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 11 Aug 2020 12:57:01 -0400 Subject: cmd/go: add basic support for overlays This CL adds basic support for listing packages with overlays. The new cmd/go/internal/fs package adds an abstraction for communicating with the file system that will open files according to their overlaid paths, and provides functions to override those in the build context to open overlaid files. There is also some support for executing builds on packages with overlays. In cmd/go/internal/work.(*Builder).build, paths are mapped to their overlaid paths before they are given as arguments to tools. For #39958 Change-Id: I5ec0eb9ebbca303e2f1e7dbe22ec32613bc1fd17 Reviewed-on: https://go-review.googlesource.com/c/go/+/253747 Trust: Michael Matloob Trust: Jay Conrod Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/work/gc.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal/work/gc.go') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index d76574932e..1f15654c79 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -18,6 +18,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/str" "cmd/internal/objabi" @@ -145,7 +146,25 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s } for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) + f := mkAbs(p.Dir, f) + + // Handle overlays. Convert path names using OverlayPath + // so these paths can be handed directly to tools. + // Deleted files won't show up in when scanning directories earlier, + // so OverlayPath will never return "" (meaning a deleted file) here. + // TODO(#39958): Handle -trimprefix and other cases where + // tools depend on the names of the files that are passed in. + // TODO(#39958): Handle cases where the package directory + // doesn't exist on disk (this can happen when all the package's + // files are in an overlay): the code expects the package directory + // to exist and runs some tools in that directory. + // TODO(#39958): Process the overlays when the + // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are + // created in (*Builder).build. Doing that requires rewriting the + // code that uses those values to expect absolute paths. + f, _ = fsys.OverlayPath(f) + + args = append(args, f) } output, err = b.runOut(a, p.Dir, nil, args...) @@ -247,6 +266,8 @@ func (a *Action) trimpath() string { } } + // TODO(#39958): Add rewrite rules for overlaid files. + return rewrite } -- cgit v1.2.1 From 21e441c461a79be41de20a99e6001098142946e6 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 14 Oct 2020 16:21:39 -0400 Subject: cmd/go: rewrite paths for overlaid files using -trimpath Pass the trimpath flag to cmd/compile to use the correct file paths for files that are overlaid: that is, the "destination" path in the overlay's Replace mapping rather than the "source" path. Also fix paths to go source files provided to the gccgo compiler. For #39958 Change-Id: I3741aeb2272bd0d5aa32cb28133b61e58264fd39 Reviewed-on: https://go-review.googlesource.com/c/go/+/257198 Trust: Michael Matloob Trust: Bryan C. Mills Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/work/gc.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'src/cmd/go/internal/work/gc.go') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 1f15654c79..56ad1872be 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -152,8 +152,6 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s // so these paths can be handed directly to tools. // Deleted files won't show up in when scanning directories earlier, // so OverlayPath will never return "" (meaning a deleted file) here. - // TODO(#39958): Handle -trimprefix and other cases where - // tools depend on the names of the files that are passed in. // TODO(#39958): Handle cases where the package directory // doesn't exist on disk (this can happen when all the package's // files are in an overlay): the code expects the package directory @@ -167,7 +165,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s args = append(args, f) } - output, err = b.runOut(a, p.Dir, nil, args...) + output, err = b.runOut(a, base.Cwd, nil, args...) return ofile, output, err } @@ -256,17 +254,28 @@ func (a *Action) trimpath() string { } rewrite := objdir + "=>" - // For "go build -trimpath", rewrite package source directory - // to a file system-independent path (just the import path). + rewriteDir := a.Package.Dir if cfg.BuildTrimpath { if m := a.Package.Module; m != nil && m.Version != "" { - rewrite += ";" + a.Package.Dir + "=>" + m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) + rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(a.Package.ImportPath, m.Path) } else { - rewrite += ";" + a.Package.Dir + "=>" + a.Package.ImportPath + rewriteDir = a.Package.ImportPath } + rewrite += ";" + a.Package.Dir + "=>" + rewriteDir } - // TODO(#39958): Add rewrite rules for overlaid files. + // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have + // same basename, so go from the overlay contents file path (passed to the compiler) + // to the path the disk path would be rewritten to. + if fsys.OverlayFile != "" { + for _, filename := range a.Package.AllFiles() { + overlayPath, ok := fsys.OverlayPath(filepath.Join(a.Package.Dir, filename)) + if !ok { + continue + } + rewrite += ";" + overlayPath + "=>" + filepath.Join(rewriteDir, filename) + } + } return rewrite } -- cgit v1.2.1 From 4d1cecdee8360ef12a817c124d7a04c9d29741c3 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 14 Oct 2020 08:06:54 -0400 Subject: cmd/dist,cmd/go: broaden use of asm macro GOEXPERIMENT_REGABI This extends a change made in https://golang.org/cl/252258 to the go command (to define an asm macro when GOEXPERIMENT=regabi is in effect); we need this same macro during the bootstrap build in order to build the runtime correctly. In addition, expand the set of packages where the macro is applied to {runtime, reflect, syscall, runtime/internal/*}, and move the logic for deciding when something is a "runtime package" out of the assembler and into cmd/{go,dist}, introducing a new assembler command line flag instead. Updates #27539, #40724. Change-Id: Ifcc7f029f56873584de1e543c55b0d3e54ad6c49 Reviewed-on: https://go-review.googlesource.com/c/go/+/262317 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/go/internal/work/gc.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'src/cmd/go/internal/work/gc.go') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 56ad1872be..2df4a52ba5 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -292,14 +292,20 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } - if p.ImportPath == "runtime" && objabi.Regabi_enabled != 0 { - // In order to make it easier to port runtime assembly - // to the register ABI, we introduce a macro - // indicating the experiment is enabled. - // - // TODO(austin): Remove this once we commit to the - // register ABI (#40724). - args = append(args, "-D=GOEXPERIMENT_REGABI=1") + if objabi.IsRuntimePackagePath(pkgpath) { + args = append(args, "-compilingRuntime") + if objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // Note: a similar change also appears in + // cmd/dist/build.go. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { -- cgit v1.2.1 From d595712540f00d980b1276ed25495ee7e05c1bfa Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 20 Oct 2020 09:06:42 -0400 Subject: cmd/asm: rename "compiling runtime" flag Rename the assembler "-compilingRuntime" flag to "-compiling-runtime", to be more consistent with the flag style of other Go commands. Change-Id: I8cc5cbf0b9b34d1dd4e9fa499d3fec8c1ef10b6e Reviewed-on: https://go-review.googlesource.com/c/go/+/263857 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/go/internal/work/gc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd/go/internal/work/gc.go') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 2df4a52ba5..e93031431c 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -293,7 +293,7 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } if objabi.IsRuntimePackagePath(pkgpath) { - args = append(args, "-compilingRuntime") + args = append(args, "-compiling-runtime") if objabi.Regabi_enabled != 0 { // In order to make it easier to port runtime assembly // to the register ABI, we introduce a macro -- cgit v1.2.1 From b08dfbaa439e4e396b979e02ea2e7d36972e8b7a Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 1 Jul 2020 16:02:42 +0000 Subject: runtime,runtime/metrics: add memory metrics This change adds support for a variety of runtime memory metrics and contains the base implementation of Read for the runtime/metrics package, which lives in the runtime. It also adds testing infrastructure for the metrics package, and a bunch of format and documentation tests. For #37112. Change-Id: I16a2c4781eeeb2de0abcb045c15105f1210e2d8a Reviewed-on: https://go-review.googlesource.com/c/go/+/247041 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Reviewed-by: Michael Pratt Trust: Michael Knyszek --- src/cmd/go/internal/work/gc.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/cmd/go/internal/work/gc.go') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index e93031431c..0c4a7fa6e3 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -89,7 +89,11 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) if p.Standard { switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": extFiles++ } } -- cgit v1.2.1