[dev.garbage] runtime: reduce stack footprint of write barriers

This is going to hurt a bit but we'll make it better later.
Now the race detector can be run again.

I added the write barrier optimizations from
CL 183020043 to try to make it hurt a little less.

TBR=rlh
CC=golang-codereviews
https://codereview.appspot.com/185070043

2 files changed, 70 insertions, 50 deletions

diff --git a/src/run.bash b/src/run.bash
index b8ce417a0..ff98d09f5 100755
--- a/src/run.bash
+++ b/src/run.bash
@@ -163,7 +163,7 @@ esac
 # and only on amd64, and only when cgo is enabled.
 # Delayed until here so we know whether to try external linking.
 # DISABLED until we get garbage collection working.
-case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED-XXX-DISABLED" in
+case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED" in
 linux-linux-amd64-1 | freebsd-freebsd-amd64-1 | darwin-darwin-amd64-1)
 	echo
 	echo '# Testing race detector.'
diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go
index 7797894fc..d6e786165 100644
--- a/src/runtime/mgc0.go
+++ b/src/runtime/mgc0.go
@@ -102,19 +102,25 @@ const (
 	_PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
 )
 
+func needwb() bool {
+	return gcphase == _GCmark || gcphase == _GCmarktermination
+}
+
 // NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
 // but if we do that, Go inserts a write barrier on *dst = src.
 //go:nosplit
 func writebarrierptr(dst *uintptr, src uintptr) {
 	*dst = src
-	writebarrierptr_nostore(dst, src)
+	if needwb() {
+		writebarrierptr_nostore(dst, src)
+	}
 }
 
 // Like writebarrierptr, but the store has already been applied.
 // Do not reapply.
 //go:nosplit
 func writebarrierptr_nostore(dst *uintptr, src uintptr) {
-	if getg() == nil { // very low-level startup
+	if getg() == nil || !needwb() { // very low-level startup
 		return
 	}
 
@@ -162,29 +168,36 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
 
 //go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
-	mask := loadPtrMask(typ)
-	nptr := typ.size / ptrSize
-	for i := uintptr(0); i < nptr; i += 2 {
-		bits := mask[i/2]
-		if (bits>>2)&_BitsMask == _BitsPointer {
-			writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
-		} else {
-			*(*uintptr)(dst) = *(*uintptr)(src)
-		}
-		dst = add(dst, ptrSize)
-		src = add(src, ptrSize)
-		if i+1 == nptr {
-			break
-		}
-		bits >>= 4
-		if (bits>>2)&_BitsMask == _BitsPointer {
-			writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
-		} else {
-			*(*uintptr)(dst) = *(*uintptr)(src)
-		}
-		dst = add(dst, ptrSize)
-		src = add(src, ptrSize)
+	if !needwb() {
+		memmove(dst, src, typ.size)
+		return
 	}
+
+	systemstack(func() {
+		mask := loadPtrMask(typ)
+		nptr := typ.size / ptrSize
+		for i := uintptr(0); i < nptr; i += 2 {
+			bits := mask[i/2]
+			if (bits>>2)&_BitsMask == _BitsPointer {
+				writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+			} else {
+				*(*uintptr)(dst) = *(*uintptr)(src)
+			}
+			dst = add(dst, ptrSize)
+			src = add(src, ptrSize)
+			if i+1 == nptr {
+				break
+			}
+			bits >>= 4
+			if (bits>>2)&_BitsMask == _BitsPointer {
+				writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+			} else {
+				*(*uintptr)(dst) = *(*uintptr)(src)
+			}
+			dst = add(dst, ptrSize)
+			src = add(src, ptrSize)
+		}
+	})
 }
 
 //go:nosplit
@@ -199,33 +212,40 @@ func writebarriercopy(typ *_type, dst, src slice) int {
 	dstp := unsafe.Pointer(dst.array)
 	srcp := unsafe.Pointer(src.array)
 
-	if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
-		// Overlap with src before dst.
-		// Copy backward, being careful not to move dstp/srcp
-		// out of the array they point into.
-		dstp = add(dstp, uintptr(n-1)*typ.size)
-		srcp = add(srcp, uintptr(n-1)*typ.size)
-		i := uint(0)
-		for {
-			writebarrierfat(typ, dstp, srcp)
-			if i++; i >= n {
-				break
+	if !needwb() {
+		memmove(dstp, srcp, uintptr(n)*typ.size)
+		return int(n)
+	}
+
+	systemstack(func() {
+		if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
+			// Overlap with src before dst.
+			// Copy backward, being careful not to move dstp/srcp
+			// out of the array they point into.
+			dstp = add(dstp, uintptr(n-1)*typ.size)
+			srcp = add(srcp, uintptr(n-1)*typ.size)
+			i := uint(0)
+			for {
+				writebarrierfat(typ, dstp, srcp)
+				if i++; i >= n {
+					break
+				}
+				dstp = add(dstp, -typ.size)
+				srcp = add(srcp, -typ.size)
 			}
-			dstp = add(dstp, -typ.size)
-			srcp = add(srcp, -typ.size)
-		}
-	} else {
-		// Copy forward, being careful not to move dstp/srcp
-		// out of the array they point into.
-		i := uint(0)
-		for {
-			writebarrierfat(typ, dstp, srcp)
-			if i++; i >= n {
-				break
+		} else {
+			// Copy forward, being careful not to move dstp/srcp
+			// out of the array they point into.
+			i := uint(0)
+			for {
+				writebarrierfat(typ, dstp, srcp)
+				if i++; i >= n {
+					break
+				}
+				dstp = add(dstp, typ.size)
+				srcp = add(srcp, typ.size)
 			}
-			dstp = add(dstp, typ.size)
-			srcp = add(srcp, typ.size)
 		}
-	}
+	})
 	return int(n)
 }