diff options
author | Russ Cox <rsc@golang.org> | 2014-11-12 14:54:31 -0500 |
---|---|---|
committer | Russ Cox <rsc@golang.org> | 2014-11-12 14:54:31 -0500 |
commit | de0da7ce7456abd9fb2ff73df0f3dccdfb6cf202 (patch) | |
tree | d6286c2595823a8fdb0015268fa893b50fbce2cb | |
parent | 7f4162b3d9492e6ae6438c78b77fdc5814d44e26 (diff) | |
download | go-de0da7ce7456abd9fb2ff73df0f3dccdfb6cf202.tar.gz |
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://codereview.appspot.com/174950043
36 files changed, 259 insertions, 422 deletions
diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 45c8e4e6b..8cdfebde5 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -200,62 +200,49 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-4 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-4 + MOVL fn+0(FP), DI // DI = fn get_tls(CX) MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m + MOVL m_gsignal(BX), DX // DX = gsignal CMPL AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVL fn+0(FP), DI // DI = fn - MOVL DI, DX - MOVL 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-4 - MOVL fn+0(FP), DI // DI = fn - get_tls(CX) - MOVL g(CX), AX // AX = g - MOVL g_m(AX), BX // BX = m + JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX - JEQ onm + JEQ noswitch MOVL m_curg(BX), BP CMPL AX, BP - JEQ oncurg + JEQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. + // Bad: g is not gsignal, not g0, not curg. What is it? // Hide call from linker nosplit analysis. - MOVL $runtime·badonm(SB), AX + MOVL $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVL $runtime·switchtoM(SB), (g_sched+gobuf_pc)(AX) + // be systemstack_switch if the G stack is scanned. + MOVL $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX) MOVL SP, (g_sched+gobuf_sp)(AX) MOVL AX, (g_sched+gobuf_g)(AX) // switch to g0 MOVL DX, g(CX) MOVL (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUBL $4, BX MOVL $runtime·mstart(SB), DX MOVL DX, 0(BX) @@ -276,8 +263,8 @@ oncurg: MOVL $0, (g_sched+gobuf_sp)(AX) RET -onm: - // already on m stack, just call directly +noswitch: + // already on system stack, just call directly MOVL DI, DX MOVL 0(DI), DI CALL DI @@ -741,7 +728,7 @@ needm: // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVL m_g0(BP), SI diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 9a74a5310..5840c32b5 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -190,55 +190,41 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-8 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-8 + MOVQ fn+0(FP), DI // DI = fn get_tls(CX) MOVQ g(CX), AX // AX = g MOVQ g_m(AX), BX // BX = m + MOVQ m_gsignal(BX), DX // DX = gsignal CMPQ AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVQ fn+0(FP), DI // DI = fn - MOVQ DI, DX - MOVQ 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-8 - MOVQ fn+0(FP), DI // DI = fn - get_tls(CX) - MOVQ g(CX), AX // AX = g - MOVQ g_m(AX), BX // BX = m + JEQ noswitch MOVQ m_g0(BX), DX // DX = g0 CMPQ AX, DX - JEQ onm + JEQ noswitch MOVQ m_curg(BX), BP CMPQ AX, BP - JEQ oncurg + JEQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. - // Hide call from linker nosplit analysis. - MOVQ $runtime·badonm(SB), AX + // Bad: g is not gsignal, not g0, not curg. What is it? + MOVQ $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVQ $runtime·switchtoM(SB), BP + // be systemstack_switch if the G stack is scanned. + MOVQ $runtime·systemstack_switch(SB), BP MOVQ BP, (g_sched+gobuf_pc)(AX) MOVQ SP, (g_sched+gobuf_sp)(AX) MOVQ AX, (g_sched+gobuf_g)(AX) @@ -246,7 +232,7 @@ oncurg: // switch to g0 MOVQ DX, g(CX) MOVQ (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUBQ $8, BX MOVQ $runtime·mstart(SB), DX MOVQ DX, 0(BX) @@ -267,7 +253,7 @@ oncurg: MOVQ $0, (g_sched+gobuf_sp)(AX) RET -onm: +noswitch: // already on m stack, just call directly MOVQ DI, DX MOVQ 0(DI), DI @@ -727,7 +713,7 @@ needm: // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVQ m_g0(BP), SI diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index 99c8569bc..a202e7ea3 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -165,55 +165,42 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-4 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of +// at the top of the system stack because the one at the top of // the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-4 + MOVL fn+0(FP), DI // DI = fn get_tls(CX) MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m + MOVL m_gsignal(BX), DX // DX = gsignal CMPL AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVL fn+0(FP), DI // DI = fn - MOVL DI, DX - MOVL 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-4 - MOVL fn+0(FP), DI // DI = fn - get_tls(CX) - MOVL g(CX), AX // AX = g - MOVL g_m(AX), BX // BX = m + JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX - JEQ onm + JEQ noswitch MOVL m_curg(BX), R8 CMPL AX, R8 - JEQ oncurg + JEQ switch // Not g0, not curg. Must be gsignal, but that's not allowed. // Hide call from linker nosplit analysis. - MOVL $runtime·badonm(SB), AX + MOVL $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVL $runtime·switchtoM(SB), SI + // be systemstack_switch if the G stack is scanned. + MOVL $runtime·systemstack_switch(SB), SI MOVL SI, (g_sched+gobuf_pc)(AX) MOVL SP, (g_sched+gobuf_sp)(AX) MOVL AX, (g_sched+gobuf_g)(AX) @@ -237,7 +224,7 @@ oncurg: MOVL $0, (g_sched+gobuf_sp)(AX) RET -onm: +noswitch: // already on m stack, just call directly MOVL DI, DX MOVL 0(DI), DI diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 897f56805..50dc4f7f2 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -191,55 +191,42 @@ TEXT runtime·mcall(SB),NOSPLIT,$-4-4 B runtime·badmcall2(SB) RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB),NOSPLIT,$0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0 MOVW $0, R0 BL (R0) // clobber lr to ensure push {lr} is kept RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $4-4 - MOVW g_m(g), R1 - MOVW m_gsignal(R1), R2 - MOVW fn+0(FP), R0 - CMP g, R2 - B.EQ ongsignal - MOVW R0, 4(R13) - BL runtime·onM(SB) - RET - -ongsignal: - MOVW R0, R7 - MOVW 0(R0), R0 - BL (R0) - RET - -// func onM(fn func()) -TEXT runtime·onM(SB),NOSPLIT,$0-4 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB),NOSPLIT,$0-4 MOVW fn+0(FP), R0 // R0 = fn MOVW g_m(g), R1 // R1 = m + MOVW m_gsignal(R1), R2 // R2 = gsignal + CMP g, R2 + B.EQ noswitch + MOVW m_g0(R1), R2 // R2 = g0 CMP g, R2 - B.EQ onm + B.EQ noswitch MOVW m_curg(R1), R3 CMP g, R3 - B.EQ oncurg + B.EQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. + // Bad: g is not gsignal, not g0, not curg. What is it? // Hide call from linker nosplit analysis. - MOVW $runtime·badonm(SB), R0 + MOVW $runtime·badsystemstack(SB), R0 BL (R0) -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVW $runtime·switchtoM(SB), R3 + // be systemstack_switch if the G stack is scanned. + MOVW $runtime·systemstack_switch(SB), R3 ADD $4, R3, R3 // get past push {lr} MOVW R3, (g_sched+gobuf_pc)(g) MOVW SP, (g_sched+gobuf_sp)(g) @@ -252,7 +239,7 @@ oncurg: BL setg<>(SB) MOVW R5, R0 MOVW (g_sched+gobuf_sp)(R2), R3 - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUB $4, R3, R3 MOVW $runtime·mstart(SB), R4 MOVW R4, 0(R3) @@ -272,7 +259,7 @@ oncurg: MOVW R3, (g_sched+gobuf_sp)(g) RET -onm: +noswitch: MOVW R0, R7 MOVW 0(R0), R0 BL (R0) @@ -567,7 +554,7 @@ TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-12 // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVW g_m(g), R8 diff --git a/src/runtime/atomic_arm.go b/src/runtime/atomic_arm.go index b1632cdd1..fd55a0aca 100644 --- a/src/runtime/atomic_arm.go +++ b/src/runtime/atomic_arm.go @@ -85,7 +85,7 @@ func atomicstore(addr *uint32, v uint32) { //go:nosplit func cas64(addr *uint64, old, new uint64) bool { var ok bool - onM(func() { + systemstack(func() { lock(addrLock(addr)) if *addr == old { *addr = new @@ -99,7 +99,7 @@ func cas64(addr *uint64, old, new uint64) bool { //go:nosplit func xadd64(addr *uint64, delta int64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr + uint64(delta) *addr = r @@ -111,7 +111,7 @@ func xadd64(addr *uint64, delta int64) uint64 { //go:nosplit func xchg64(addr *uint64, v uint64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr *addr = v @@ -123,7 +123,7 @@ func xchg64(addr *uint64, v uint64) uint64 { //go:nosplit func atomicload64(addr *uint64) uint64 { var r uint64 - onM(func() { + systemstack(func() { lock(addrLock(addr)) r = *addr unlock(addrLock(addr)) @@ -133,7 +133,7 @@ func atomicload64(addr *uint64) uint64 { //go:nosplit func atomicstore64(addr *uint64, v uint64) { - onM(func() { + systemstack(func() { lock(addrLock(addr)) *addr = v unlock(addrLock(addr)) diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index a1fc06d39..258cabfc8 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -103,7 +103,7 @@ func cgocall_errno(fn, arg unsafe.Pointer) int32 { // Create an extra M for callbacks on threads not created by Go on first cgo call. if needextram == 1 && cas(&needextram, 1, 0) { - onM(newextram) + systemstack(newextram) } /* @@ -195,7 +195,7 @@ func cgocallbackg1() { gp := getg() if gp.m.needextram { gp.m.needextram = false - onM(newextram) + systemstack(newextram) } // Add entry to defer stack in case of panic. diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index 245eaebda..d56678e21 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -102,9 +102,9 @@ var ( ) func setcpuprofilerate(hz int32) { - g := getg() - g.m.scalararg[0] = uintptr(hz) - onM(setcpuprofilerate_m) + systemstack(func() { + setcpuprofilerate_m(hz) + }) } // lostProfileData is a no-op function used in profiles diff --git a/src/runtime/debug.go b/src/runtime/debug.go index f954265aa..105b79cfe 100644 --- a/src/runtime/debug.go +++ b/src/runtime/debug.go @@ -25,14 +25,14 @@ func GOMAXPROCS(n int) int { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) // newprocs will be processed by starttheworld newprocs = int32(n) gp.m.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) return ret } diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 0ecf91fdf..e871236e4 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -59,21 +59,21 @@ func parforiters_m() func NewParFor(nthrmax uint32) *ParFor { var desc *ParFor - onM(func() { + systemstack(func() { desc = (*ParFor)(unsafe.Pointer(parforalloc(nthrmax))) }) return desc } func ParForSetup(desc *ParFor, nthr, n uint32, ctx *byte, wait bool, body func(*ParFor, uint32)) { - onM(func() { + systemstack(func() { parforsetup((*parfor)(unsafe.Pointer(desc)), nthr, n, unsafe.Pointer(ctx), wait, *(*func(*parfor, uint32))(unsafe.Pointer(&body))) }) } func ParForDo(desc *ParFor) { - onM(func() { + systemstack(func() { parfordo((*parfor)(unsafe.Pointer(desc))) }) } @@ -87,7 +87,7 @@ func ParForIters(desc *ParFor, tid uint32) (uint32, uint32) { func GCMask(x interface{}) (ret []byte) { e := (*eface)(unsafe.Pointer(&x)) s := (*slice)(unsafe.Pointer(&ret)) - onM(func() { + systemstack(func() { var len uintptr getgcmask(e.data, e._type, &s.array, &len) s.len = uint(len) @@ -97,10 +97,10 @@ func GCMask(x interface{}) (ret []byte) { } func RunSchedLocalQueueTest() { - onM(testSchedLocalQueue) + systemstack(testSchedLocalQueue) } func RunSchedLocalQueueStealTest() { - onM(testSchedLocalQueueSteal) + systemstack(testSchedLocalQueueSteal) } var HaveGoodHash = haveGoodHash @@ -121,7 +121,7 @@ func GogoBytes() int32 { // entry point for testing func GostringW(w []uint16) (s string) { - onM(func() { + systemstack(func() { s = gostringw(&w[0]) }) return diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 6cc5df810..34fdeb2b4 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -112,7 +112,8 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) { if xpc > f.entry && (g == nil || g.entry != funcPC(sigpanic)) { xpc-- } - line = int(funcline(f, xpc, &file)) + file, line32 := funcline(f, xpc) + line = int(line32) ok = true return } diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index 01e70a34d..c942e0163 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -602,8 +602,7 @@ func dumpmemprof_callback(b *bucket, nstk uintptr, pstk *uintptr, size, allocs, if i > 0 && pc > f.entry { pc-- } - var file string - line := funcline(f, pc, &file) + file, line := funcline(f, pc) dumpstr(file) dumpint(uint64(line)) } @@ -657,11 +656,8 @@ func mdump() { flush() } -func writeheapdump_m() { +func writeheapdump_m(fd uintptr) { _g_ := getg() - fd := _g_.m.scalararg[0] - _g_.m.scalararg[0] = 0 - casgstatus(_g_.m.curg, _Grunning, _Gwaiting) _g_.waitreason = "dumping heap" diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index a11724500..89b6ffac3 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -57,7 +57,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // This function must be atomic wrt GC, but for performance reasons // we don't acquirem/releasem on fast path. The code below does not have // split stack checks, so it can't be preempted by GC. - // Functions like roundup/add are inlined. And onM/racemalloc are nosplit. + // Functions like roundup/add are inlined. And systemstack/racemalloc are nosplit. // If debugMalloc = true, these assumptions are checked below. if debugMalloc { mp := acquirem() @@ -143,7 +143,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { s = c.alloc[tinySizeClass] v := s.freelist if v == nil { - onM(func() { + systemstack(func() { mCache_Refill(c, tinySizeClass) }) s = c.alloc[tinySizeClass] @@ -173,7 +173,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { s = c.alloc[sizeclass] v := s.freelist if v == nil { - onM(func() { + systemstack(func() { mCache_Refill(c, int32(sizeclass)) }) s = c.alloc[sizeclass] @@ -193,7 +193,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { c.local_cachealloc += intptr(size) } else { var s *mspan - onM(func() { + systemstack(func() { s = largeAlloc(size, uint32(flags)) }) x = unsafe.Pointer(uintptr(s.start << pageShift)) @@ -247,22 +247,17 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // into the GC bitmap. It's 7 times slower than copying // from the pre-unrolled mask, but saves 1/16 of type size // memory for the mask. - mp := acquirem() - mp.ptrarg[0] = x - mp.ptrarg[1] = unsafe.Pointer(typ) - mp.scalararg[0] = uintptr(size) - mp.scalararg[1] = uintptr(size0) - onM(unrollgcproginplace_m) - releasem(mp) + systemstack(func() { + unrollgcproginplace_m(x, typ, size, size0) + }) goto marked } ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) // Check whether the program is already unrolled. if uintptr(atomicloadp(unsafe.Pointer(ptrmask)))&0xff == 0 { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(typ) - onM(unrollgcprog_m) - releasem(mp) + systemstack(func() { + unrollgcprog_m(typ) + }) } ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte } else { @@ -434,7 +429,7 @@ func gogc(force int32) { mp = acquirem() mp.gcing = 1 releasem(mp) - onM(stoptheworld) + systemstack(stoptheworld) if mp != acquirem() { gothrow("gogc: rescheduled") } @@ -455,20 +450,16 @@ func gogc(force int32) { startTime = nanotime() } // switch to g0, call gc, then switch back - mp.scalararg[0] = uintptr(uint32(startTime)) // low 32 bits - mp.scalararg[1] = uintptr(startTime >> 32) // high 32 bits - if force >= 2 { - mp.scalararg[2] = 1 // eagersweep - } else { - mp.scalararg[2] = 0 - } - onM(gc_m) + eagersweep := force >= 2 + systemstack(func() { + gc_m(startTime, eagersweep) + }) } // all done mp.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) releasem(mp) mp = nil @@ -580,8 +571,8 @@ func SetFinalizer(obj interface{}, finalizer interface{}) { f := (*eface)(unsafe.Pointer(&finalizer)) ftyp := f._type if ftyp == nil { - // switch to M stack and remove finalizer - onM(func() { + // switch to system stack and remove finalizer + systemstack(func() { removefinalizer(e.data) }) return @@ -628,7 +619,7 @@ okarg: // make sure we have a finalizer goroutine createfing() - onM(func() { + systemstack(func() { if !addfinalizer(e.data, (*funcval)(f.data), nret, fint, ot) { gothrow("runtime.SetFinalizer: finalizer already set") } diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index 7482bc047..d3afef6be 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -35,7 +35,7 @@ func allocmcache() *mcache { } func freemcache(c *mcache) { - onM(func() { + systemstack(func() { mCache_ReleaseAll(c) stackcache_clear(c) gcworkbuffree(c.gcworkbuf) diff --git a/src/runtime/mem.go b/src/runtime/mem.go index 6bd250d38..183567251 100644 --- a/src/runtime/mem.go +++ b/src/runtime/mem.go @@ -82,15 +82,16 @@ func ReadMemStats(m *MemStats) { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) - gp.m.ptrarg[0] = noescape(unsafe.Pointer(m)) - onM(readmemstats_m) + systemstack(func() { + readmemstats_m(m) + }) gp.m.gcing = 0 gp.m.locks++ semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) gp.m.locks-- } @@ -99,14 +100,15 @@ func writeHeapDump(fd uintptr) { semacquire(&worldsema, false) gp := getg() gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) - gp.m.scalararg[0] = fd - onM(writeheapdump_m) + systemstack(func() { + writeheapdump_m(fd) + }) gp.m.gcing = 0 gp.m.locks++ semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) gp.m.locks-- } diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 569bf5ddc..0bb735355 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1058,7 +1058,7 @@ func sweepone() uintptr { func gosweepone() uintptr { var ret uintptr - onM(func() { + systemstack(func() { ret = sweepone() }) return ret @@ -1152,7 +1152,7 @@ func updatememstats(stats *gcstats) { } // Flush MCache's to MCentral. - onM(flushallmcaches) + systemstack(flushallmcaches) // Aggregate local stats. cachestats() @@ -1193,13 +1193,6 @@ func updatememstats(stats *gcstats) { memstats.heap_objects = memstats.nmalloc - memstats.nfree } -// Structure of arguments passed to function gc(). -// This allows the arguments to be passed via mcall. -type gc_args struct { - start_time int64 // start time of GC in ns (just before stoptheworld) - eagersweep bool -} - func gcinit() { if unsafe.Sizeof(workbuf{}) != _WorkbufSize { gothrow("runtime: size of Workbuf is suboptimal") @@ -1211,21 +1204,18 @@ func gcinit() { gcbssmask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcbss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss))) } -func gc_m() { +func gc_m(start_time int64, eagersweep bool) { _g_ := getg() gp := _g_.m.curg casgstatus(gp, _Grunning, _Gwaiting) gp.waitreason = "garbage collection" - var a gc_args - a.start_time = int64(_g_.m.scalararg[0]) | int64(uintptr(_g_.m.scalararg[1]))<<32 - a.eagersweep = _g_.m.scalararg[2] != 0 - gc(&a) + gc(start_time, eagersweep) if nbadblock > 0 { // Work out path from root to bad block. for { - gc(&a) + gc(start_time, eagersweep) if nbadblock >= int32(len(badblock)) { gothrow("cannot find path to bad pointer") } @@ -1235,7 +1225,7 @@ func gc_m() { casgstatus(gp, _Gwaiting, _Grunning) } -func gc(args *gc_args) { +func gc(start_time int64, eagersweep bool) { if _DebugGCPtrs { print("GC start\n") } @@ -1246,8 +1236,8 @@ func gc(args *gc_args) { _g_ := getg() _g_.m.traceback = 2 - t0 := args.start_time - work.tstart = args.start_time + t0 := start_time + work.tstart = start_time var t1 int64 if debug.gctrace > 0 { @@ -1367,7 +1357,7 @@ func gc(args *gc_args) { sweep.spanidx = 0 unlock(&mheap_.lock) - if _ConcurrentSweep && !args.eagersweep { + if _ConcurrentSweep && !eagersweep { lock(&gclock) if !sweep.started { go bgsweep() @@ -1394,11 +1384,7 @@ func gc(args *gc_args) { } } -func readmemstats_m() { - _g_ := getg() - stats := (*mstats)(_g_.m.ptrarg[0]) - _g_.m.ptrarg[0] = nil - +func readmemstats_m(stats *MemStats) { updatememstats(nil) // Size of the trailing by_size array differs between Go and C, @@ -1406,14 +1392,14 @@ func readmemstats_m() { memmove(unsafe.Pointer(stats), unsafe.Pointer(&memstats), sizeof_C_MStats) // Stack numbers are part of the heap numbers, separate those out for user consumption - stats.stacks_sys = stats.stacks_inuse - stats.heap_inuse -= stats.stacks_inuse - stats.heap_sys -= stats.stacks_inuse + stats.StackSys = stats.StackInuse + stats.HeapInuse -= stats.StackInuse + stats.HeapSys -= stats.StackInuse } //go:linkname readGCStats runtime/debug.readGCStats func readGCStats(pauses *[]uint64) { - onM(func() { + systemstack(func() { readGCStats_m(pauses) }) } @@ -1578,16 +1564,7 @@ func unrollglobgcprog(prog *byte, size uintptr) bitvector { return bitvector{int32(masksize * 8), &mask[0]} } -func unrollgcproginplace_m() { - _g_ := getg() - - v := _g_.m.ptrarg[0] - typ := (*_type)(_g_.m.ptrarg[1]) - size := _g_.m.scalararg[0] - size0 := _g_.m.scalararg[1] - _g_.m.ptrarg[0] = nil - _g_.m.ptrarg[1] = nil - +func unrollgcproginplace_m(v unsafe.Pointer, typ *_type, size, size0 uintptr) { pos := uintptr(0) prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1]))) for pos != size0 { @@ -1613,12 +1590,7 @@ func unrollgcproginplace_m() { var unroll mutex // Unrolls GC program in typ.gc[1] into typ.gc[0] -func unrollgcprog_m() { - _g_ := getg() - - typ := (*_type)(_g_.m.ptrarg[0]) - _g_.m.ptrarg[0] = nil - +func unrollgcprog_m(typ *_type) { lock(&unroll) mask := (*byte)(unsafe.Pointer(uintptr(typ.gc[0]))) if *mask == 0 { diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go index f7e01c898..6d4ae61c1 100644 --- a/src/runtime/mgc0.go +++ b/src/runtime/mgc0.go @@ -28,7 +28,7 @@ func gc_unixnanotime(now *int64) { func freeOSMemory() { gogc(2) // force GC and do eager sweep - onM(scavenge_m) + systemstack(scavenge_m) } var poolcleanup func() diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index b451b631c..fedcd69c5 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -174,7 +174,7 @@ func mHeap_Reclaim(h *mheap, npage uintptr) { func mHeap_Alloc_m(h *mheap, npage uintptr, sizeclass int32, large bool) *mspan { _g_ := getg() if _g_ != _g_.m.g0 { - gothrow("_mheap_alloc not on M stack") + gothrow("_mheap_alloc not on g0 stack") } lock(&h.lock) @@ -226,7 +226,7 @@ func mHeap_Alloc(h *mheap, npage uintptr, sizeclass int32, large bool, needzero // It might trigger stack growth, and the stack growth code needs // to be able to allocate heap. var s *mspan - onM(func() { + systemstack(func() { s = mHeap_Alloc_m(h, npage, sizeclass, large) }) @@ -242,7 +242,7 @@ func mHeap_Alloc(h *mheap, npage uintptr, sizeclass int32, large bool, needzero func mHeap_AllocStack(h *mheap, npage uintptr) *mspan { _g_ := getg() if _g_ != _g_.m.g0 { - gothrow("mheap_allocstack not on M stack") + gothrow("mheap_allocstack not on g0 stack") } lock(&h.lock) s := mHeap_AllocSpanLocked(h, npage) @@ -428,7 +428,7 @@ func mHeap_LookupMaybe(h *mheap, v unsafe.Pointer) *mspan { // Free the span back into the heap. func mHeap_Free(h *mheap, s *mspan, acct int32) { - onM(func() { + systemstack(func() { mp := getg().m lock(&h.lock) memstats.heap_alloc += uint64(mp.mcache.local_cachealloc) @@ -447,7 +447,7 @@ func mHeap_Free(h *mheap, s *mspan, acct int32) { func mHeap_FreeStack(h *mheap, s *mspan) { _g_ := getg() if _g_ != _g_.m.g0 { - gothrow("mheap_freestack not on M stack") + gothrow("mheap_freestack not on g0 stack") } s.needzero = 1 lock(&h.lock) diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index 6ff33743b..ba989b1b8 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -244,7 +244,7 @@ func mProf_Malloc(p unsafe.Pointer, size uintptr) { // This reduces potential contention and chances of deadlocks. // Since the object must be alive during call to mProf_Malloc, // it's fine to do this non-atomically. - onM(func() { + systemstack(func() { setprofilebucket(p, b) }) } @@ -523,7 +523,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { gp := getg() semacquire(&worldsema, false) gp.m.gcing = 1 - onM(stoptheworld) + systemstack(stoptheworld) n = NumGoroutine() if n <= len(p) { @@ -531,7 +531,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { r := p sp := getcallersp(unsafe.Pointer(&p)) pc := getcallerpc(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { saveg(pc, sp, gp, &r[0]) }) r = r[1:] @@ -546,7 +546,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { gp.m.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) } return n, ok @@ -570,7 +570,7 @@ func Stack(buf []byte, all bool) int { semacquire(&worldsema, false) mp.gcing = 1 releasem(mp) - onM(stoptheworld) + systemstack(stoptheworld) if mp != acquirem() { gothrow("Stack: rescheduled") } @@ -580,7 +580,7 @@ func Stack(buf []byte, all bool) int { if len(buf) > 0 { sp := getcallersp(unsafe.Pointer(&buf)) pc := getcallerpc(unsafe.Pointer(&buf)) - onM(func() { + systemstack(func() { g0 := getg() g0.writebuf = buf[0:0:len(buf)] goroutineheader(gp) @@ -596,7 +596,7 @@ func Stack(buf []byte, all bool) int { if all { mp.gcing = 0 semrelease(&worldsema) - onM(starttheworld) + systemstack(starttheworld) } releasem(mp) return n @@ -619,7 +619,7 @@ func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) { goroutineheader(gp) pc := getcallerpc(unsafe.Pointer(&p)) sp := getcallersp(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { traceback(pc, sp, 0, gp) }) } else { @@ -639,7 +639,7 @@ func tracefree(p unsafe.Pointer, size uintptr) { goroutineheader(gp) pc := getcallerpc(unsafe.Pointer(&p)) sp := getcallersp(unsafe.Pointer(&p)) - onM(func() { + systemstack(func() { traceback(pc, sp, 0, gp) }) print("\n") diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index e11623c1b..dd00b2a96 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -72,7 +72,7 @@ type pollCache struct { var pollcache pollCache func netpollServerInit() { - onM(netpollinit) + systemstack(netpollinit) } func netpollOpen(fd uintptr) (*pollDesc, int) { @@ -94,7 +94,7 @@ func netpollOpen(fd uintptr) (*pollDesc, int) { unlock(&pd.lock) var errno int32 - onM(func() { + systemstack(func() { errno = netpollopen(fd, pd) }) return pd, int(errno) @@ -110,7 +110,7 @@ func netpollClose(pd *pollDesc) { if pd.rg != 0 && pd.rg != pdReady { gothrow("netpollClose: blocked read on closing descriptor") } - onM(func() { + systemstack(func() { netpollclose(uintptr(pd.fd)) }) pollcache.free(pd) @@ -143,7 +143,7 @@ func netpollWait(pd *pollDesc, mode int) int { } // As for now only Solaris uses level-triggered IO. if GOOS == "solaris" { - onM(func() { + systemstack(func() { netpollarm(pd, mode) }) } diff --git a/src/runtime/os1_darwin.go b/src/runtime/os1_darwin.go index b30ffbe83..2fbf2cae0 100644 --- a/src/runtime/os1_darwin.go +++ b/src/runtime/os1_darwin.go @@ -24,7 +24,7 @@ func semawakeup(mp *m) { //go:nosplit func semacreate() uintptr { var x uintptr - onM(func() { + systemstack(func() { x = uintptr(mach_semcreate()) }) return x @@ -349,7 +349,7 @@ func semasleep1(ns int64) int32 { //go:nosplit func semasleep(ns int64) int32 { var r int32 - onM(func() { + systemstack(func() { r = semasleep1(ns) }) return r @@ -368,9 +368,9 @@ func mach_semrelease(sem uint32) { // mach_semrelease must be completely nosplit, // because it is called from Go code. - // If we're going to die, start that process on the m stack + // If we're going to die, start that process on the system stack // to avoid a Go stack split. - onM_signalok(func() { macherror(r, "semaphore_signal") }) + systemstack(func() { macherror(r, "semaphore_signal") }) } } diff --git a/src/runtime/os1_freebsd.go b/src/runtime/os1_freebsd.go index 0b2fedad9..dd22b61d6 100644 --- a/src/runtime/os1_freebsd.go +++ b/src/runtime/os1_freebsd.go @@ -32,7 +32,7 @@ func getncpu() int32 { //go:nosplit func futexsleep(addr *uint32, val uint32, ns int64) { - onM(func() { + systemstack(func() { futexsleep1(addr, val, ns) }) } @@ -60,7 +60,7 @@ func futexwakeup(addr *uint32, cnt uint32) { return } - onM(func() { + systemstack(func() { print("umtx_wake_addr=", addr, " ret=", ret, "\n") }) } diff --git a/src/runtime/os1_linux.go b/src/runtime/os1_linux.go index 311d0ab1a..cbbd2d689 100644 --- a/src/runtime/os1_linux.go +++ b/src/runtime/os1_linux.go @@ -58,7 +58,7 @@ func futexwakeup(addr *uint32, cnt uint32) { // I don't know that futex wakeup can return // EAGAIN or EINTR, but if it does, it would be // safe to loop and call futex again. - onM_signalok(func() { + systemstack(func() { print("futexwakeup addr=", addr, " returned ", ret, "\n") }) diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 9f3b9a3aa..8debd3356 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -55,8 +55,8 @@ func throwinit() { //go:nosplit func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn if getg().m.curg != getg() { - // go code on the m stack can't defer - gothrow("defer on m") + // go code on the system stack can't defer + gothrow("defer on system stack") } // the arguments of fn are in a perilous state. The stack map @@ -71,7 +71,7 @@ func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn } callerpc := getcallerpc(unsafe.Pointer(&siz)) - onM(func() { + systemstack(func() { d := newdefer(siz) if d._panic != nil { gothrow("deferproc: d.panic != nil after newdefer") @@ -322,7 +322,7 @@ func gopanic(e interface{}) { print("panic: ") printany(e) print("\n") - gothrow("panic on m stack") + gothrow("panic on system stack") } // m.softfloat is set during software floating point. @@ -470,17 +470,17 @@ func gorecover(argp uintptr) interface{} { //go:nosplit func startpanic() { - onM_signalok(startpanic_m) + systemstack(startpanic_m) } //go:nosplit func dopanic(unused int) { + pc := getcallerpc(unsafe.Pointer(&unused)) + sp := getcallersp(unsafe.Pointer(&unused)) gp := getg() - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(gp) - mp.scalararg[0] = getcallerpc((unsafe.Pointer)(&unused)) - mp.scalararg[1] = getcallersp((unsafe.Pointer)(&unused)) - onM_signalok(dopanic_m) // should never return + systemstack(func() { + dopanic_m(gp, pc, sp) // should never return + }) *(*int)(nil) = 0 } diff --git a/src/runtime/panic1.go b/src/runtime/panic1.go index 4c387485f..17eadb413 100644 --- a/src/runtime/panic1.go +++ b/src/runtime/panic1.go @@ -94,20 +94,13 @@ func startpanic_m() { var didothers bool var deadlock mutex -func dopanic_m() { - _g_ := getg() - - gp := (*g)(_g_.m.ptrarg[0]) - _g_.m.ptrarg[0] = nil - pc := uintptr(_g_.m.scalararg[0]) - sp := uintptr(_g_.m.scalararg[1]) - _g_.m.scalararg[1] = 0 - +func dopanic_m(gp *g, pc, sp uintptr) { if gp.sig != 0 { print("[signal ", hex(gp.sig), " code=", hex(gp.sigcode0), " addr=", hex(gp.sigcode1), " pc=", hex(gp.sigpc), "]\n") } var docrash bool + _g_ := getg() if t := gotraceback(&docrash); t > 0 { if gp != gp.m.g0 { print("\n") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 140717535..05ecb3d9e 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -27,7 +27,7 @@ func main() { maxstacksize = 250000000 } - onM(newsysmon) + systemstack(newsysmon) // Lock the main goroutine onto this, the main OS thread, // during initialization. Most programs won't care, but a few @@ -151,7 +151,7 @@ func goparkunlock(lock *mutex, reason string) { } func goready(gp *g) { - onM(func() { + systemstack(func() { ready(gp) }) } diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go index 319c8e6ee..a19bf144c 100644 --- a/src/runtime/proc1.go +++ b/src/runtime/proc1.go @@ -362,7 +362,7 @@ func castogscanstatus(gp *g, oldval, newval uint32) bool { //go:nosplit func casgstatus(gp *g, oldval, newval uint32) { if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { - onM(func() { + systemstack(func() { print("casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") gothrow("casgstatus: bad incoming values") }) @@ -374,7 +374,7 @@ func casgstatus(gp *g, oldval, newval uint32) { // Help GC if needed. if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) { gp.preemptscan = false - onM(func() { + systemstack(func() { gcphasework(gp) }) } @@ -1573,8 +1573,8 @@ func save(pc, sp uintptr) { // because we do not know which of the uintptr arguments are // really pointers (back into the stack). // In practice, this means that we make the fast path run through -// entersyscall doing no-split things, and the slow path has to use onM -// to run bigger things on the m stack. +// entersyscall doing no-split things, and the slow path has to use systemstack +// to run bigger things on the system stack. // // reentersyscall is the entry point used by cgo callbacks, where explicitly // saved SP and PC are restored. This is needed when exitsyscall will be called @@ -1602,11 +1602,11 @@ func reentersyscall(pc, sp uintptr) { _g_.syscallpc = pc casgstatus(_g_, _Grunning, _Gsyscall) if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { - onM(entersyscall_bad) + systemstack(entersyscall_bad) } if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic - onM(entersyscall_sysmon) + systemstack(entersyscall_sysmon) save(pc, sp) } @@ -1614,7 +1614,7 @@ func reentersyscall(pc, sp uintptr) { _g_.m.p.m = nil atomicstore(&_g_.m.p.status, _Psyscall) if sched.gcwaiting != 0 { - onM(entersyscall_gcwait) + systemstack(entersyscall_gcwait) save(pc, sp) } @@ -1674,10 +1674,10 @@ func entersyscallblock(dummy int32) { _g_.syscallpc = _g_.sched.pc casgstatus(_g_, _Grunning, _Gsyscall) if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { - onM(entersyscall_bad) + systemstack(entersyscall_bad) } - onM(entersyscallblock_handoff) + systemstack(entersyscallblock_handoff) // Resave for traceback during blocked call. save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) @@ -1768,18 +1768,18 @@ func exitsyscallfast() bool { // Try to get any other idle P. _g_.m.p = nil if sched.pidle != nil { - onM(exitsyscallfast_pidle) - if _g_.m.scalararg[0] != 0 { - _g_.m.scalararg[0] = 0 + var ok bool + systemstack(func() { + ok = exitsyscallfast_pidle() + }) + if ok { return true } } return false } -func exitsyscallfast_pidle() { - _g_ := getg() - +func exitsyscallfast_pidle() bool { lock(&sched.lock) _p_ := pidleget() if _p_ != nil && atomicload(&sched.sysmonwait) != 0 { @@ -1789,10 +1789,9 @@ func exitsyscallfast_pidle() { unlock(&sched.lock) if _p_ != nil { acquirep(_p_) - _g_.m.scalararg[0] = 1 - } else { - _g_.m.scalararg[0] = 0 + return true } + return false } // exitsyscall slow path on g0. @@ -1844,7 +1843,7 @@ func beforefork() { // Called from syscall package before fork. //go:nosplit func syscall_BeforeFork() { - onM(beforefork) + systemstack(beforefork) } func afterfork() { @@ -1863,7 +1862,7 @@ func afterfork() { // Called from syscall package after fork in parent. //go:nosplit func syscall_AfterFork() { - onM(afterfork) + systemstack(afterfork) } // Allocate a new g, with a stack big enough for stacksize bytes. @@ -1871,7 +1870,7 @@ func malg(stacksize int32) *g { newg := allocg() if stacksize >= 0 { stacksize = round2(_StackSystem + stacksize) - onM(func() { + systemstack(func() { newg.stack = stackalloc(uint32(stacksize)) }) newg.stackguard0 = newg.stack.lo + _StackGuard @@ -1894,7 +1893,7 @@ func newproc(siz int32, fn *funcval) { } pc := getcallerpc(unsafe.Pointer(&siz)) - onM(func() { + systemstack(func() { newproc1(fn, (*uint8)(argp), siz, 0, pc) }) } @@ -2037,7 +2036,7 @@ retry: _p_.gfreecnt-- if gp.stack.lo == 0 { // Stack was deallocated in gfput. Allocate a new one. - onM(func() { + systemstack(func() { gp.stack = stackalloc(_FixedStack) }) gp.stackguard0 = gp.stack.lo + _StackGuard @@ -2121,7 +2120,7 @@ func UnlockOSThread() { func unlockOSThread() { _g_ := getg() if _g_.m.locked < _LockInternal { - onM(badunlockosthread) + systemstack(badunlockosthread) } _g_.m.locked -= _LockInternal dounlockOSThread() @@ -2307,12 +2306,7 @@ func sigprof(pc *uint8, sp *uint8, lr *uint8, gp *g, mp *m) { } // Arrange to call fn with a traceback hz times a second. -func setcpuprofilerate_m() { - _g_ := getg() - - hz := int32(_g_.m.scalararg[0]) - _g_.m.scalararg[0] = 0 - +func setcpuprofilerate_m(hz int32) { // Force sane arguments. if hz < 0 { hz = 0 @@ -2320,6 +2314,7 @@ func setcpuprofilerate_m() { // Disable preemption, otherwise we can be rescheduled to another thread // that has profiling enabled. + _g_ := getg() _g_.m.locks++ // Stop profiler on this thread so that it is safe to lock prof. diff --git a/src/runtime/race.go b/src/runtime/race.go index 7d38fae1c..649cd7295 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -80,8 +80,8 @@ func racesymbolize(ctx *symbolizeContext) { } ctx.fn = funcname(f) - var file string - ctx.line = uintptr(funcline(f, ctx.pc, &file)) + file, line := funcline(f, ctx.pc) + ctx.line = uintptr(line) ctx.file = &bytes(file)[0] // assume NUL-terminated ctx.off = ctx.pc - f.entry ctx.res = 1 diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 81782ebfc..c999b3072 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -245,8 +245,6 @@ type m struct { traceback uint8 waitunlockf unsafe.Pointer // todo go func(*g, unsafe.pointer) bool waitlock unsafe.Pointer - scalararg [4]uintptr // scalar argument/return for mcall - ptrarg [4]unsafe.Pointer // pointer argument/return for mcall //#ifdef GOOS_windows thread uintptr // thread handle // these are here because they are too large to be on the stack diff --git a/src/runtime/signal.go b/src/runtime/signal.go deleted file mode 100644 index 8bfd82c2a..000000000 --- a/src/runtime/signal.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -func sigenable_m() { - _g_ := getg() - sigenable(uint32(_g_.m.scalararg[0])) -} - -func sigdisable_m() { - _g_ := getg() - sigdisable(uint32(_g_.m.scalararg[0])) -} diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index ee2ee2e75..c457083dc 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -7,5 +7,5 @@ package runtime func os_sigpipe() { - onM(sigpipe) + systemstack(sigpipe) } diff --git a/src/runtime/sigqueue.go b/src/runtime/sigqueue.go index b2155ce86..82ead228f 100644 --- a/src/runtime/sigqueue.go +++ b/src/runtime/sigqueue.go @@ -139,7 +139,7 @@ func signal_enable(s uint32) { return } sig.wanted[s/32] |= 1 << (s & 31) - sigenable_go(s) + sigenable(s) } // Must only be called from a single goroutine at a time. @@ -148,7 +148,7 @@ func signal_disable(s uint32) { return } sig.wanted[s/32] &^= 1 << (s & 31) - sigdisable_go(s) + sigdisable(s) } // This runs on a foreign stack, without an m or a g. No stack split. @@ -156,15 +156,3 @@ func signal_disable(s uint32) { func badsignal(sig uintptr) { cgocallback(unsafe.Pointer(funcPC(sigsend)), noescape(unsafe.Pointer(&sig)), unsafe.Sizeof(sig)) } - -func sigenable_go(s uint32) { - g := getg() - g.m.scalararg[0] = uintptr(s) - onM(sigenable_m) -} - -func sigdisable_go(s uint32) { - g := getg() - g.m.scalararg[0] = uintptr(s) - onM(sigdisable_m) -} diff --git a/src/runtime/softfloat_arm.go b/src/runtime/softfloat_arm.go index d806d1f04..746b9ea21 100644 --- a/src/runtime/softfloat_arm.go +++ b/src/runtime/softfloat_arm.go @@ -606,7 +606,7 @@ done: //go:nosplit func _sfloat2(pc uint32, regs *[15]uint32) { - onM(func() { + systemstack(func() { pc = sfloat2(pc, regs) }) } diff --git a/src/runtime/stack1.go b/src/runtime/stack1.go index f93160b15..40dfc76a6 100644 --- a/src/runtime/stack1.go +++ b/src/runtime/stack1.go @@ -418,8 +418,8 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { if stackDebug >= 2 { print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") } - if f.entry == switchtoMPC { - // A special routine at the bottom of stack of a goroutine that does an onM call. + if f.entry == systemstack_switchPC { + // A special routine at the bottom of stack of a goroutine that does an systemstack call. // We will allow it to be copied even though we don't // have full GC info for it (because it is written in asm). return true @@ -801,7 +801,7 @@ func shrinkfinish() { //go:nosplit func morestackc() { - onM(func() { + systemstack(func() { gothrow("attempt to execute C code on Go stack") }) } diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 2be4ad531..bca0a1d7c 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -38,56 +38,28 @@ func getg() *g //go:noescape func mcall(fn func(*g)) -// onM switches from the g to the g0 stack and invokes fn(). -// When fn returns, onM switches back to the g and returns, -// continuing execution on the g stack. -// If arguments must be passed to fn, they can be written to -// g->m->ptrarg (pointers) and g->m->scalararg (non-pointers) -// before the call and then consulted during fn. -// Similarly, fn can pass return values back in those locations. -// If fn is written in Go, it can be a closure, which avoids the need for -// ptrarg and scalararg entirely. -// After reading values out of ptrarg and scalararg it is conventional -// to zero them to avoid (memory or information) leaks. +// systemstack runs fn on a system stack. +// If systemstack is called from the per-OS-thread (g0) stack, or +// if systemstack is called from the signal handling (gsignal) stack, +// systemstack calls fn directly and returns. +// Otherwise, systemstack is being called from the limited stack +// of an ordinary goroutine. In this case, systemstack switches +// to the per-OS-thread stack, calls fn, and switches back. +// It is common to use a func literal as the argument, in order +// to share inputs and outputs with the code around the call +// to system stack: // -// If onM is called from a g0 stack, it invokes fn and returns, -// without any stack switches. -// -// If onM is called from a gsignal stack, it crashes the program. -// The implication is that functions used in signal handlers must -// not use onM. -// -// NOTE(rsc): We could introduce a separate onMsignal that is -// like onM but if called from a gsignal stack would just run fn on -// that stack. The caller of onMsignal would be required to save the -// old values of ptrarg/scalararg and restore them when the call -// was finished, in case the signal interrupted an onM sequence -// in progress on the g or g0 stacks. Until there is a clear need for this, -// we just reject onM in signal handling contexts entirely. -// -//go:noescape -func onM(fn func()) - -// onMsignal is like onM but is allowed to be used in code that -// might run on the gsignal stack. Code running on a signal stack -// may be interrupting an onM sequence on the main stack, so -// if the onMsignal calling sequence writes to ptrarg/scalararg, -// it must first save the old values and then restore them when -// finished. As an exception to the rule, it is fine not to save and -// restore the values if the program is trying to crash rather than -// return from the signal handler. -// Once all the runtime is written in Go, there will be no ptrarg/scalararg -// and the distinction between onM and onMsignal (and perhaps mcall) -// can go away. -// -// If onMsignal is called from a gsignal stack, it invokes fn directly, -// without a stack switch. Otherwise onMsignal behaves like onM. +// ... set up y ... +// systemstack(func() { +// x = bigcall(y) +// }) +// ... use x ... // //go:noescape -func onM_signalok(fn func()) +func systemstack(fn func()) -func badonm() { - gothrow("onM called from signal goroutine") +func badsystemstack() { + gothrow("systemstack called from unexpected goroutine") } // memclr clears n bytes starting at ptr. @@ -272,4 +244,4 @@ func call268435456(fn, arg unsafe.Pointer, n, retoffset uint32) func call536870912(fn, arg unsafe.Pointer, n, retoffset uint32) func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32) -func switchtoM() +func systemstack_switch() diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index e11fd128a..749a289cd 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -121,8 +121,8 @@ func (f *Func) Entry() uintptr { func (f *Func) FileLine(pc uintptr) (file string, line int) { // Pass strict=false here, because anyone can call this function, // and they might just be wrong about targetpc belonging to f. - line = int(funcline1(f.raw(), pc, &file, false)) - return file, line + file, line32 := funcline1(f.raw(), pc, false) + return file, int(line32) } func findfunc(pc uintptr) *_func { @@ -207,20 +207,19 @@ func gofuncname(f *_func) string { return gostringnocopy(funcname(f)) } -func funcline1(f *_func, targetpc uintptr, file *string, strict bool) int32 { - *file = "?" +func funcline1(f *_func, targetpc uintptr, strict bool) (file string, line int32) { fileno := int(pcvalue(f, f.pcfile, targetpc, strict)) - line := pcvalue(f, f.pcln, targetpc, strict) + line = pcvalue(f, f.pcln, targetpc, strict) if fileno == -1 || line == -1 || fileno >= len(filetab) { // print("looking for ", hex(targetpc), " in ", gofuncname(f), " got file=", fileno, " line=", lineno, "\n") - return 0 + return "?", 0 } - *file = gostringnocopy(&pclntable[filetab[fileno]]) - return line + file = gostringnocopy(&pclntable[filetab[fileno]]) + return } -func funcline(f *_func, targetpc uintptr, file *string) int32 { - return funcline1(f, targetpc, file, true) +func funcline(f *_func, targetpc uintptr) (file string, line int32) { + return funcline1(f, targetpc, true) } func funcspdelta(f *_func, targetpc uintptr) int32 { diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 78678815c..e1cc9123f 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -32,16 +32,16 @@ const usesLR = GOARCH != "amd64" && GOARCH != "amd64p32" && GOARCH != "386" var ( // initialized in tracebackinit - deferprocPC uintptr - goexitPC uintptr - jmpdeferPC uintptr - mcallPC uintptr - morestackPC uintptr - mstartPC uintptr - newprocPC uintptr - rt0_goPC uintptr - sigpanicPC uintptr - switchtoMPC uintptr + deferprocPC uintptr + goexitPC uintptr + jmpdeferPC uintptr + mcallPC uintptr + morestackPC uintptr + mstartPC uintptr + newprocPC uintptr + rt0_goPC uintptr + sigpanicPC uintptr + systemstack_switchPC uintptr externalthreadhandlerp uintptr // initialized elsewhere ) @@ -60,7 +60,7 @@ func tracebackinit() { newprocPC = funcPC(newproc) rt0_goPC = funcPC(rt0_go) sigpanicPC = funcPC(sigpanic) - switchtoMPC = funcPC(switchtoM) + systemstack_switchPC = funcPC(systemstack_switch) } // Traceback over the deferred function calls. @@ -337,8 +337,7 @@ func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf print(hex(argp[i])) } print(")\n") - var file string - line := funcline(f, tracepc, &file) + file, line := funcline(f, tracepc) print("\t", file, ":", line) if frame.pc > f.entry { print(" +", hex(frame.pc-f.entry)) @@ -482,8 +481,7 @@ func printcreatedby(gp *g) { if pc > f.entry { tracepc -= _PCQuantum } - var file string - line := funcline(f, tracepc, &file) + file, line := funcline(f, tracepc) print("\t", file, ":", line) if pc > f.entry { print(" +", hex(pc-f.entry)) @@ -530,7 +528,7 @@ func callers(skip int, pcbuf *uintptr, m int) int { sp := getcallersp(unsafe.Pointer(&skip)) pc := uintptr(getcallerpc(unsafe.Pointer(&skip))) var n int - onM(func() { + systemstack(func() { n = gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) }) return n |