cmd/compile, cmd/link: record lengths in object file

Record total number of relocations, pcdata, automatics, funcdata and files in
object file and use these numbers in the linker to allocate contiguous
slices to later be filled by the defined symbols.

name       old secs    new secs    delta
LinkCmdGo   0.52 ± 3%   0.49 ± 3%  -4.21%   (p=0.000 n=91+92)
LinkJuju    4.48 ± 4%   4.21 ± 7%  -6.08%  (p=0.000 n=96+100)

name       old MaxRSS  new MaxRSS  delta
LinkCmdGo   122k ± 2%   120k ± 4%  -1.66%   (p=0.000 n=98+93)
LinkJuju    799k ± 5%   865k ± 8%  +8.29%   (p=0.000 n=89+99)

GOGC=off

name       old secs    new secs    delta
LinkCmdGo   0.42 ± 2%   0.41 ± 0%  -2.98%    (p=0.000 n=89+70)
LinkJuju    3.61 ± 0%   3.52 ± 1%  -2.46%    (p=0.000 n=80+89)

name       old MaxRSS  new MaxRSS  delta
LinkCmdGo   130k ± 1%   128k ± 1%  -1.33%  (p=0.000 n=100+100)
LinkJuju   1.00M ± 0%  0.99M ± 0%  -1.70%  (p=0.000 n=100+100)

Change-Id: Ie08f6ccd4311bb78d8950548c678230a58635c73
Reviewed-on: https://go-review.googlesource.com/21026
Reviewed-by: David Crawshaw <crawshaw@golang.org>
Run-TryBot: David Crawshaw <crawshaw@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

3 files changed, 143 insertions, 34 deletions

diff --git a/src/cmd/internal/goobj/read.go b/src/cmd/internal/goobj/read.go
index 2506f07de2..d64f4cbce1 100644
--- a/src/cmd/internal/goobj/read.go
+++ b/src/cmd/internal/goobj/read.go
@@ -614,6 +614,12 @@ func (r *objReader) parseObject(prefix []byte) error {
 	}
 
 	dataLength := r.readInt()
+	r.readInt() // n relocations - ignore
+	r.readInt() // n pcdata - ignore
+	r.readInt() // n autom - ignore
+	r.readInt() // n funcdata - ignore
+	r.readInt() // n files - ignore
+
 	r.dataOffset = r.offset
 	r.skip(int64(dataLength))
 
diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go
index b9eb8014ec..aba832f27b 100644
--- a/src/cmd/internal/obj/objfile.go
+++ b/src/cmd/internal/obj/objfile.go
@@ -21,7 +21,13 @@
 //	- empty string (marks end of sequence)
 //	- sequence of symbol references used by the defined symbols
 //	- byte 0xff (marks end of sequence)
-//	- integer (length of following data)
+//	- sequence of integer lengths:
+//		- total data length
+//		- total number of relocations
+//		- total number of pcdata
+//		- total number of automatics
+//		- total number of funcdata
+//		- total number of files
 //	- data, the content of the defined symbols
 //	- sequence of defined symbols
 //	- byte 0xff (marks end of sequence)
@@ -303,6 +309,54 @@ func flushplist(ctxt *Link, freeProgs bool) {
 	}
 }
 
+type sectionLengths struct {
+	data     int
+	reloc    int
+	pcdata   int
+	autom    int
+	funcdata int
+	file     int
+}
+
+func (l *sectionLengths) add(s *LSym) {
+	l.data += len(s.P)
+	l.reloc += len(s.R)
+
+	if s.Type != STEXT {
+		return
+	}
+
+	pc := s.Pcln
+
+	data := 0
+	data += len(pc.Pcsp.P)
+	data += len(pc.Pcfile.P)
+	data += len(pc.Pcline.P)
+	for i := 0; i < len(pc.Pcdata); i++ {
+		data += len(pc.Pcdata[i].P)
+	}
+
+	l.data += data
+	l.pcdata += len(pc.Pcdata)
+
+	autom := 0
+	for a := s.Autom; a != nil; a = a.Link {
+		autom++
+	}
+	l.autom += autom
+	l.funcdata += len(pc.Funcdataoff)
+	l.file += len(pc.File)
+}
+
+func wrlengths(b *Biobuf, sl sectionLengths) {
+	wrint(b, int64(sl.data))
+	wrint(b, int64(sl.reloc))
+	wrint(b, int64(sl.pcdata))
+	wrint(b, int64(sl.autom))
+	wrint(b, int64(sl.funcdata))
+	wrint(b, int64(sl.file))
+}
+
 func Writeobjfile(ctxt *Link, b *Biobuf) {
 	// Emit header.
 	Bputc(b, 0)
@@ -317,28 +371,22 @@ func Writeobjfile(ctxt *Link, b *Biobuf) {
 	}
 	wrstring(b, "")
 
-	var dataLength int64
+	var lengths sectionLengths
+
 	// Emit symbol references.
 	for _, s := range ctxt.Text {
 		writerefs(ctxt, b, s)
-		dataLength += int64(len(s.P))
-
-		pc := s.Pcln
-		dataLength += int64(len(pc.Pcsp.P))
-		dataLength += int64(len(pc.Pcfile.P))
-		dataLength += int64(len(pc.Pcline.P))
-		for i := 0; i < len(pc.Pcdata); i++ {
-			dataLength += int64(len(pc.Pcdata[i].P))
-		}
+		lengths.add(s)
 	}
 	for _, s := range ctxt.Data {
 		writerefs(ctxt, b, s)
-		dataLength += int64(len(s.P))
+		lengths.add(s)
 	}
 	Bputc(b, 0xff)
 
+	wrlengths(b, lengths)
+
 	// Write data block
-	wrint(b, dataLength)
 	for _, s := range ctxt.Text {
 		b.w.Write(s.P)
 		pc := s.Pcln
diff --git a/src/cmd/link/internal/ld/objfile.go b/src/cmd/link/internal/ld/objfile.go
index 2e8f01099c..b9121c6f81 100644
--- a/src/cmd/link/internal/ld/objfile.go
+++ b/src/cmd/link/internal/ld/objfile.go
@@ -21,9 +21,15 @@ package ld
 //	- byte 1 - version number
 //	- sequence of strings giving dependencies (imported packages)
 //	- empty string (marks end of sequence)
-//	- sequence of sybol references used by the defined symbols
+//	- sequence of symbol references used by the defined symbols
 //	- byte 0xff (marks end of sequence)
-//	- integer (length of following data)
+//	- sequence of integer lengths:
+//		- total data length
+//		- total number of relocations
+//		- total number of pcdata
+//		- total number of automatics
+//		- total number of funcdata
+//		- total number of files
 //	- data, the content of the defined symbols
 //	- sequence of defined symbols
 //	- byte 0xff (marks end of sequence)
@@ -149,9 +155,9 @@ func ldobjfile(ctxt *Link, f *obj.Biobuf, pkg string, length int64, pn string) {
 		readref(ctxt, f, pkg, pn)
 	}
 
-	dataLength := rdint64(f)
-	data := make([]byte, dataLength)
-	obj.Bread(f, data)
+	sl := rdslices(f)
+
+	obj.Bread(f, sl.data)
 
 	for {
 		c, err := f.Peek(1)
@@ -161,7 +167,7 @@ func ldobjfile(ctxt *Link, f *obj.Biobuf, pkg string, length int64, pn string) {
 		if c[0] == 0xff {
 			break
 		}
-		readsym(ctxt, f, &data, pkg, pn)
+		readsym(ctxt, f, sl, pkg, pn)
 	}
 
 	buf = [8]uint8{}
@@ -177,9 +183,38 @@ func ldobjfile(ctxt *Link, f *obj.Biobuf, pkg string, length int64, pn string) {
 
 var dupSym = &LSym{Name: ".dup"}
 
-func readsym(ctxt *Link, f *obj.Biobuf, buf *[]byte, pkg string, pn string) {
+type slices struct {
+	data        []byte
+	reloc       []Reloc
+	pcdata      []Pcdata
+	autom       []Auto
+	funcdata    []*LSym
+	funcdataoff []int64
+	file        []*LSym
+}
+
+func rdslices(f *obj.Biobuf) *slices {
+	sl := &slices{}
+
+	n := rdint(f)
+	sl.data = make([]byte, n)
+	n = rdint(f)
+	sl.reloc = make([]Reloc, n)
+	n = rdint(f)
+	sl.pcdata = make([]Pcdata, n)
+	n = rdint(f)
+	sl.autom = make([]Auto, n)
+	n = rdint(f)
+	sl.funcdata = make([]*LSym, n)
+	sl.funcdataoff = make([]int64, n)
+	n = rdint(f)
+	sl.file = make([]*LSym, n)
+	return sl
+}
+
+func readsym(ctxt *Link, f *obj.Biobuf, sl *slices, pkg string, pn string) {
 	if obj.Bgetc(f) != 0xfe {
-		log.Fatalf("readsym out of sync")
+		log.Fatalln("readsym out of sync")
 	}
 	t := rdint(f)
 	s := rdsym(ctxt, f, pkg)
@@ -188,8 +223,9 @@ func readsym(ctxt *Link, f *obj.Biobuf, buf *[]byte, pkg string, pn string) {
 	local := flags&2 != 0
 	size := rdint(f)
 	typ := rdsym(ctxt, f, pkg)
-	data := rddata(f, buf)
+	data := rddata(f, &sl.data)
 	nreloc := rdint(f)
+	isdup := false
 
 	var dup *LSym
 	if s.Type != 0 && s.Type != obj.SXREF {
@@ -212,6 +248,7 @@ func readsym(ctxt *Link, f *obj.Biobuf, buf *[]byte, pkg string, pn string) {
 		if len(s.P) > 0 {
 			dup = s
 			s = dupSym
+			isdup = true
 		}
 	}
 
@@ -237,12 +274,16 @@ overwrite:
 	if typ != nil {
 		s.Gotype = typ
 	}
-	if dup != nil && typ != nil { // if bss sym defined multiple times, take type from any one def
+	if isdup && typ != nil { // if bss sym defined multiple times, take type from any one def
 		dup.Gotype = typ
 	}
 	s.P = data
 	if nreloc > 0 {
-		s.R = make([]Reloc, nreloc)
+		s.R = sl.reloc[:nreloc:nreloc]
+		if !isdup {
+			sl.reloc = sl.reloc[nreloc:]
+		}
+
 		var r *Reloc
 		for i := 0; i < nreloc; i++ {
 			r = &s.R[i]
@@ -265,7 +306,11 @@ overwrite:
 			s.Attr |= AttrReflectMethod
 		}
 		n := rdint(f)
-		s.Autom = make([]Auto, n)
+		s.Autom = sl.autom[:n:n]
+		if !isdup {
+			sl.autom = sl.autom[n:]
+		}
+
 		for i := 0; i < n; i++ {
 			s.Autom[i] = Auto{
 				Asym:    rdsym(ctxt, f, pkg),
@@ -277,17 +322,24 @@ overwrite:
 
 		s.Pcln = new(Pcln)
 		pc := s.Pcln
-		pc.Pcsp.P = rddata(f, buf)
-		pc.Pcfile.P = rddata(f, buf)
-		pc.Pcline.P = rddata(f, buf)
+		pc.Pcsp.P = rddata(f, &sl.data)
+		pc.Pcfile.P = rddata(f, &sl.data)
+		pc.Pcline.P = rddata(f, &sl.data)
 		n = rdint(f)
-		pc.Pcdata = make([]Pcdata, n)
+		pc.Pcdata = sl.pcdata[:n:n]
+		if !isdup {
+			sl.pcdata = sl.pcdata[n:]
+		}
 		for i := 0; i < n; i++ {
-			pc.Pcdata[i].P = rddata(f, buf)
+			pc.Pcdata[i].P = rddata(f, &sl.data)
 		}
 		n = rdint(f)
-		pc.Funcdata = make([]*LSym, n)
-		pc.Funcdataoff = make([]int64, n)
+		pc.Funcdata = sl.funcdata[:n:n]
+		pc.Funcdataoff = sl.funcdataoff[:n:n]
+		if !isdup {
+			sl.funcdata = sl.funcdata[n:]
+			sl.funcdataoff = sl.funcdataoff[n:]
+		}
 		for i := 0; i < n; i++ {
 			pc.Funcdata[i] = rdsym(ctxt, f, pkg)
 		}
@@ -295,12 +347,15 @@ overwrite:
 			pc.Funcdataoff[i] = rdint64(f)
 		}
 		n = rdint(f)
-		pc.File = make([]*LSym, n)
+		pc.File = sl.file[:n:n]
+		if !isdup {
+			sl.file = sl.file[n:]
+		}
 		for i := 0; i < n; i++ {
 			pc.File[i] = rdsym(ctxt, f, pkg)
 		}
 
-		if dup == nil {
+		if !isdup {
 			if s.Attr.OnList() {
 				log.Fatalf("symbol %s listed multiple times", s.Name)
 			}