diff options
Diffstat (limited to 'src/runtime/mgc0.go')
| -rw-r--r-- | src/runtime/mgc0.go | 118 |
1 files changed, 69 insertions, 49 deletions
diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go index 7797894fc..d6e786165 100644 --- a/src/runtime/mgc0.go +++ b/src/runtime/mgc0.go @@ -102,19 +102,25 @@ const ( _PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1) ) +func needwb() bool { + return gcphase == _GCmark || gcphase == _GCmarktermination +} + // NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer, // but if we do that, Go inserts a write barrier on *dst = src. //go:nosplit func writebarrierptr(dst *uintptr, src uintptr) { *dst = src - writebarrierptr_nostore(dst, src) + if needwb() { + writebarrierptr_nostore(dst, src) + } } // Like writebarrierptr, but the store has already been applied. // Do not reapply. //go:nosplit func writebarrierptr_nostore(dst *uintptr, src uintptr) { - if getg() == nil { // very low-level startup + if getg() == nil || !needwb() { // very low-level startup return } @@ -162,29 +168,36 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) { //go:nosplit func writebarrierfat(typ *_type, dst, src unsafe.Pointer) { - mask := loadPtrMask(typ) - nptr := typ.size / ptrSize - for i := uintptr(0); i < nptr; i += 2 { - bits := mask[i/2] - if (bits>>2)&_BitsMask == _BitsPointer { - writebarrierptr((*uintptr)(dst), *(*uintptr)(src)) - } else { - *(*uintptr)(dst) = *(*uintptr)(src) - } - dst = add(dst, ptrSize) - src = add(src, ptrSize) - if i+1 == nptr { - break - } - bits >>= 4 - if (bits>>2)&_BitsMask == _BitsPointer { - writebarrierptr((*uintptr)(dst), *(*uintptr)(src)) - } else { - *(*uintptr)(dst) = *(*uintptr)(src) - } - dst = add(dst, ptrSize) - src = add(src, ptrSize) + if !needwb() { + memmove(dst, src, typ.size) + return } + + systemstack(func() { + mask := loadPtrMask(typ) + nptr := typ.size / ptrSize + for i := uintptr(0); i < nptr; i += 2 { + bits := mask[i/2] + if (bits>>2)&_BitsMask == _BitsPointer { + writebarrierptr((*uintptr)(dst), *(*uintptr)(src)) + } else { + *(*uintptr)(dst) = *(*uintptr)(src) + } + dst = add(dst, ptrSize) + src = add(src, ptrSize) + if i+1 == nptr { + break + } + bits >>= 4 + if (bits>>2)&_BitsMask == _BitsPointer { + writebarrierptr((*uintptr)(dst), *(*uintptr)(src)) + } else { + *(*uintptr)(dst) = *(*uintptr)(src) + } + dst = add(dst, ptrSize) + src = add(src, ptrSize) + } + }) } //go:nosplit @@ -199,33 +212,40 @@ func writebarriercopy(typ *_type, dst, src slice) int { dstp := unsafe.Pointer(dst.array) srcp := unsafe.Pointer(src.array) - if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) { - // Overlap with src before dst. - // Copy backward, being careful not to move dstp/srcp - // out of the array they point into. - dstp = add(dstp, uintptr(n-1)*typ.size) - srcp = add(srcp, uintptr(n-1)*typ.size) - i := uint(0) - for { - writebarrierfat(typ, dstp, srcp) - if i++; i >= n { - break + if !needwb() { + memmove(dstp, srcp, uintptr(n)*typ.size) + return int(n) + } + + systemstack(func() { + if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) { + // Overlap with src before dst. + // Copy backward, being careful not to move dstp/srcp + // out of the array they point into. + dstp = add(dstp, uintptr(n-1)*typ.size) + srcp = add(srcp, uintptr(n-1)*typ.size) + i := uint(0) + for { + writebarrierfat(typ, dstp, srcp) + if i++; i >= n { + break + } + dstp = add(dstp, -typ.size) + srcp = add(srcp, -typ.size) } - dstp = add(dstp, -typ.size) - srcp = add(srcp, -typ.size) - } - } else { - // Copy forward, being careful not to move dstp/srcp - // out of the array they point into. - i := uint(0) - for { - writebarrierfat(typ, dstp, srcp) - if i++; i >= n { - break + } else { + // Copy forward, being careful not to move dstp/srcp + // out of the array they point into. + i := uint(0) + for { + writebarrierfat(typ, dstp, srcp) + if i++; i >= n { + break + } + dstp = add(dstp, typ.size) + srcp = add(srcp, typ.size) } - dstp = add(dstp, typ.size) - srcp = add(srcp, typ.size) } - } + }) return int(n) } |